inst-jobs 2.4.4 → 2.4.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90e78e46299d111959e7e48f25fe4e05a7939c6fb54fcd423deff4b960e79e6c
-  data.tar.gz: 20963910cd2cb1856f73cabbdc1547131a0d5096e26f7cc444f0a6e7d26b0ded
+  metadata.gz: 8e17453e7b2cfd39920dfb36c615bcd813ecf3416985e405e26d5062716ccaa7
+  data.tar.gz: 17ad240a2304ca6ee648308199ac8d83d53498ee93cd130bf3cbd86dde729673
 SHA512:
-  metadata.gz: 22155dba3fd9f1201e67930be8a3cc488da21b02b2066b59fe9a0f28f5670393534ddbb87a937979df356ba602c23a27e97f7650463fa2eb7c53e76647df5f02
-  data.tar.gz: 41a6bec7f263657aac33b62eeb8169db1898dfdcdbfadc32d91a985f54cb037a5691d75f02dff80b9c80ea57387a746fa70e6710545fe6b9c12724dde9f8f720
+  metadata.gz: 29ed31003455122f52a8f45ac22751acf3f71213e4f93fae280425d90f5a35919ef881220660c453469904cec107383d0764afa0d87458da3b4fdded272c5c78
+  data.tar.gz: a9cb6ff603ad8c683b326cebfdb24acef31a450f3e6fab88af3760adf8f726e4c4dc28797bbcb7bb5e38b0f2fef0af97ee3c145a8031db723e5c67bb9c94d385

data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -38,6 +38,16 @@ module Delayed
             job = new(attributes, &block)
             job.single_step_create(on_conflict: on_conflict)
           end
+
+          def attempt_advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
+          end
+
+          def advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.execute("SELECT pg_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
+          end
         end
 
         def single_step_create(on_conflict: nil)
@@ -458,9 +468,19 @@ module Delayed
           where("locked_by LIKE ?", "#{name}:%").pluck(:locked_by).map { |locked_by| locked_by.split(":").last.to_i }
         end
 
+        def self.prefetch_jobs_lock_name
+          "Delayed::Job.unlock_orphaned_prefetched_jobs"
+        end
+
         def self.unlock_orphaned_prefetched_jobs
-          horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
-          where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+          transaction do
+            # for db performance reasons, we only need one process doing this at a time
+            # so if we can't get an advisory lock, just abort. we'll try again soon
+            return unless attempt_advisory_lock(prefetch_jobs_lock_name)
+
+            horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
+            where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+          end
         end
 
         def self.unlock(jobs)

data/lib/delayed/backend/base.rb

@@ -166,17 +166,36 @@ module Delayed
           pid_regex = pid || '(\d+)'
           regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
           unlocked_jobs = 0
+          escaped_name = name.gsub("\\", "\\\\")
+                             .gsub("%", "\\%")
+                             .gsub("_", "\\_")
+          locked_by_like = "#{escaped_name}:%"
           running = false if pid
-          running_jobs.each do |job|
-            next unless job.locked_by =~ regex
-
-            unless pid
-              job_pid = $1.to_i
-              running = Process.kill(0, job_pid) rescue false
-            end
-            unless running
-              unlocked_jobs += 1
-              job.reschedule("process died")
+          jobs = running_jobs.limit(100)
+          jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
+          ignores = []
+          loop do
+            batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
+            batch = batch_scope.to_a
+            break if batch.empty?
+
+            batch.each do |job|
+              unless job.locked_by =~ regex
+                ignores << job.id
+                next
+              end
+
+              unless pid
+                job_pid = $1.to_i
+                running = Process.kill(0, job_pid) rescue false
+              end
+
+              if running
+                ignores << job.id
+              else
+                unlocked_jobs += 1
+                job.reschedule("process died")
+              end
             end
           end
           unlocked_jobs

data/lib/delayed/periodic.rb

@@ -33,7 +33,13 @@ module Delayed
       # we used to queue up a job in a strand here, and perform the audit inside that job
       # however, now that we're using singletons for scheduling periodic jobs,
       # it's fine to just do the audit in-line here without risk of creating duplicates
-      perform_audit!
+      Delayed::Job.transaction do
+        # for db performance reasons, we only need one process doing this at a time
+        # so if we can't get an advisory lock, just abort. we'll try again soon
+        return unless Delayed::Job.attempt_advisory_lock("Delayed::Periodic#audit_queue")
+
+        perform_audit!
+      end
     end
 
     # make sure all periodic jobs are scheduled for their next run in the job queue

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "2.4.4"
+  VERSION = "2.4.9"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -195,11 +195,13 @@ module Delayed
         def unlock_timed_out_prefetched_jobs
           @prefetched_jobs.each do |(worker_config, jobs)|
             next if jobs.empty?
+            next unless jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
 
-            if jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
+            Delayed::Job.transaction do
+              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
               Delayed::Job.unlock(jobs)
-              @prefetched_jobs[worker_config] = []
             end
+            @prefetched_jobs[worker_config] = []
           end
         end
 
@@ -207,7 +209,10 @@ module Delayed
           @prefetched_jobs.each do |(_worker_config, jobs)|
             next if jobs.empty?
 
-            Delayed::Job.unlock(jobs)
+            Delayed::Job.transaction do
+              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
+              Delayed::Job.unlock(jobs)
+            end
           end
           @prefetched_jobs = {}
         end

data/lib/delayed/worker/health_check.rb

@@ -28,14 +28,16 @@ module Delayed
           Delayed::Job.transaction do
             # this action is a special case, and SHOULD NOT be a periodic job
             # because if it gets wiped out suddenly during execution
-            # it can't go clean up it's abandoned self.  Therefore,
+            # it can't go clean up its abandoned self.  Therefore,
             # we expect it to get run from it's own process forked from the job pool
             # and we try to get an advisory lock when it runs.  If we succeed,
             # no other worker is trying to do this right now (and if we abandon the
             # operation, the transaction will end, releasing the advisory lock).
-            result = attempt_advisory_lock
+            result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
             return unless result
 
+            horizon = 5.minutes.ago
+
             checker = Worker::HealthCheck.build(
               type: Settings.worker_health_check_type,
               config: Settings.worker_health_check_config,
@@ -43,13 +45,16 @@ module Delayed
             )
             live_workers = checker.live_workers
 
-            Delayed::Job.running_jobs.each do |job|
-              # prefetched jobs have their own way of automatically unlocking themselves
-              next if job.locked_by.start_with?("prefetch:")
-
-              next if live_workers.include?(job.locked_by)
+            loop do
+              batch = Delayed::Job.running_jobs
+                                  .where("locked_at<?", horizon)
+                                  .where.not("locked_by LIKE 'prefetch:%'")
+                                  .where.not(locked_by: live_workers)
+                                  .limit(100)
+                                  .to_a
+              break if batch.empty?
 
-              begin
+              batch.each do |job|
                 Delayed::Job.transaction do
                   # double check that the job is still there. locked_by will immediately be reset
                   # to nil in this transaction by Job#reschedule
@@ -59,19 +64,12 @@ module Delayed
 
                   job.reschedule
                 end
-              rescue
-                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
+            rescue
+              ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
             end
           end
         end
-
-        def attempt_advisory_lock
-          lock_name = "Delayed::Worker::HealthCheck#reschedule_abandoned_jobs"
-          conn = ActiveRecord::Base.connection
-          fn_name = conn.quote_table_name("half_md5_as_bigint")
-          conn.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
-        end
       end
 
       attr_accessor :config, :worker_name

data/spec/active_record_job_spec.rb

@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
       expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
     end
 
-    it "doesn't  allow a second worker to get exclusive access if failed to be processed by worker1 and
-        run_at time is now in future (due to backing off behaviour)" do
+    it "doesn't allow a second worker to get exclusive access if failed to be " \
+       "processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
       @job.update(attempts: 1, run_at: 1.day.from_now)
       expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
     end

data/spec/delayed/daemon_spec.rb

@@ -3,9 +3,10 @@
 require "spec_helper"
 
 RSpec.describe Delayed::Daemon do
+  subject { described_class.new(pid_folder) }
+
   let(:pid_folder) { "/test/pid/folder" }
   let(:pid) { 9999 }
-  let(:subject) { described_class.new(pid_folder) }
 
   before do
     allow(subject).to receive(:pid).and_return(pid)

data/spec/delayed/work_queue/in_process_spec.rb

@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
     Delayed::Worker.lifecycle.reset!
   end
 
-  let(:subject) { described_class.new }
   let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
   let(:args) { ["worker_name", worker_config] }
 

data/spec/delayed/work_queue/parent_process/client_spec.rb

@@ -3,7 +3,8 @@
 require "spec_helper"
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
-  let(:subject) { described_class.new(addrinfo).tap(&:init) }
+  subject { described_class.new(addrinfo).tap(&:init) }
+
   let(:addrinfo) { double("Addrinfo") }
   let(:connection) { double("Socket") }
   let(:job) { Delayed::Job.new(locked_by: "worker_name") }

data/spec/delayed/work_queue/parent_process/server_spec.rb

@@ -15,8 +15,9 @@ class JobClass
 end
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
+  subject { described_class.new(listen_socket) }
+
   let(:parent) { Delayed::WorkQueue::ParentProcess.new }
-  let(:subject) { described_class.new(listen_socket) }
   let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
   let(:job) { JobClass.new }
   let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }

data/spec/delayed/work_queue/parent_process_spec.rb

@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
     Delayed::Worker.lifecycle.reset!
   end
 
-  let(:subject) { described_class.new }
-
   describe "#initalize(config = Settings.parent_process)" do
     it "must expand a relative path to be within the Rails root" do
       queue = described_class.new("server_address" => "tmp/foo.sock")

data/spec/delayed/worker/health_check_spec.rb

@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
       end
     end
 
-    let(:initial_run_at) { Time.zone.now }
+    let(:initial_run_at) { 10.minutes.ago }
 
     before do
       klass.live_workers = %w[alive]
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
 
     it "ignores jobs that are re-locked after fetching from db" do
       Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
-      allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
+      # we need to return @dead_job itself, which doesn't match the database
+      jobs_scope = double
+      allow(jobs_scope).to receive(:where).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:not).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
+      allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.locked_by).to eq "someone_else"
@@ -104,14 +110,14 @@ RSpec.describe Delayed::Worker::HealthCheck do
 
     it "ignores jobs that are prefetched" do
       Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
-      allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
+      allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.locked_by).to eq "prefetch:some_node"
     end
 
     it "bails immediately if advisory lock already taken" do
-      allow(described_class).to receive(:attempt_advisory_lock).and_return(false)
+      allow(Delayed::Job).to receive(:attempt_advisory_lock).and_return(false)
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.run_at.to_i).to eq(initial_run_at.to_i)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 2.4.4
+  version: 2.4.9
 platform: ruby
 authors:
 - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-18 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -432,6 +432,7 @@ files:
 - db/migrate/20210809145804_add_n_strand_index.rb
 - db/migrate/20210812210128_add_singleton_column.rb
 - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
+- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb