inst-jobs 2.4.6 → 2.4.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ba509dbe247e504537397a8482595ee6dbeac623c69141a24aa56f753343484f
-  data.tar.gz: 69502011608a6ac485012dd8ebc5ed9e616b8643ed92a297778100f9ddf51e15
+  metadata.gz: 65052c093825ca46de07a054fc2b62925cf2d1cb0a3fcb877b408e5c88edc865
+  data.tar.gz: '08d1c103f2c41f11f5989a0b5e2bb857e7d8ea250ba995dcd1ddae62510426cf'
 SHA512:
-  metadata.gz: bba4b9a818f3ce3f0cae2f9d77a556c2d9770a47730328d7b3d6de5c8e5bc338ab24da4a4c5006d68375c9b8c24f24af04a0e711ee71d57b17526c0ca401e275
-  data.tar.gz: 8ccc1674c9d7f7b1caa27d759a34b9670bc298ece8d85472ba23b83e03cb672cff7372547a84d8c13363aa93447ff0249255f02e5353458d252ef77e1bc89b35
+  metadata.gz: 3e28511d5c23fcdccd976e9d8f9f09ae318c50154d01223b84b3f60de42e34dbb55ed6735618224865b8c8050e15eed78bd9c8505b85abb56515c41c4266c985
+  data.tar.gz: bc60ce604e1b5077251c1c5855a3bd0582950c03c0ed0423b0daf25e8750d6e2f6dfbb32b18e99ace156a1a5232ac8e827377997185bd8c4889abe20e6e7249b

data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -43,6 +43,11 @@ module Delayed
             fn_name = connection.quote_table_name("half_md5_as_bigint")
             connection.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
           end
+
+          def advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.execute("SELECT pg_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
+          end
         end
 
         def single_step_create(on_conflict: nil)

data/lib/delayed/backend/base.rb

@@ -166,17 +166,36 @@ module Delayed
           pid_regex = pid || '(\d+)'
           regex = Regexp.new("^#{Regexp.escape(name)}:#{pid_regex}$")
           unlocked_jobs = 0
+          escaped_name = name.gsub("\\", "\\\\")
+                             .gsub("%", "\\%")
+                             .gsub("_", "\\_")
+          locked_by_like = "#{escaped_name}:%"
           running = false if pid
-          running_jobs.each do |job|
-            next unless job.locked_by =~ regex
-
-            unless pid
-              job_pid = $1.to_i
-              running = Process.kill(0, job_pid) rescue false
-            end
-            unless running
-              unlocked_jobs += 1
-              job.reschedule("process died")
+          jobs = running_jobs.limit(100)
+          jobs = pid ? jobs.where(locked_by: "#{name}:#{pid}") : jobs.where("locked_by LIKE ?", locked_by_like)
+          ignores = []
+          loop do
+            batch_scope = ignores.empty? ? jobs : jobs.where.not(id: ignores)
+            batch = batch_scope.to_a
+            break if batch.empty?
+
+            batch.each do |job|
+              unless job.locked_by =~ regex
+                ignores << job.id
+                next
+              end
+
+              unless pid
+                job_pid = $1.to_i
+                running = Process.kill(0, job_pid) rescue false
+              end
+
+              if running
+                ignores << job.id
+              else
+                unlocked_jobs += 1
+                job.reschedule("process died")
+              end
             end
           end
           unlocked_jobs

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "2.4.6"
+  VERSION = "2.4.10"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -179,7 +179,7 @@ module Delayed
             end
 
             jobs_to_send.each do |(recipient, job_to_send)|
-              @waiting_clients[worker_config].delete(client)
+              @waiting_clients[worker_config].delete(recipient)
               begin
                 logger.debug("Sending job #{job_to_send.id} to #{recipient.name}")
                 client_timeout { Marshal.dump(job_to_send, recipient.socket) }
@@ -198,7 +198,7 @@ module Delayed
             next unless jobs.first.locked_at < Time.now.utc - Settings.parent_process[:prefetched_jobs_timeout]
 
             Delayed::Job.transaction do
-              Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock('#{Delayed::Job.prefetch_jobs_lock_name}')")
+              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
               Delayed::Job.unlock(jobs)
             end
             @prefetched_jobs[worker_config] = []
@@ -210,7 +210,7 @@ module Delayed
             next if jobs.empty?
 
             Delayed::Job.transaction do
-              Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock('#{Delayed::Job.prefetch_jobs_lock_name}')")
+              Delayed::Job.advisory_lock(Delayed::Job.prefetch_jobs_lock_name)
               Delayed::Job.unlock(jobs)
             end
           end

data/lib/delayed/worker/health_check.rb

@@ -28,7 +28,7 @@ module Delayed
           Delayed::Job.transaction do
             # this action is a special case, and SHOULD NOT be a periodic job
             # because if it gets wiped out suddenly during execution
-            # it can't go clean up it's abandoned self.  Therefore,
+            # it can't go clean up its abandoned self.  Therefore,
             # we expect it to get run from it's own process forked from the job pool
             # and we try to get an advisory lock when it runs.  If we succeed,
             # no other worker is trying to do this right now (and if we abandon the
@@ -36,6 +36,8 @@ module Delayed
             result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
             return unless result
 
+            horizon = 5.minutes.ago
+
             checker = Worker::HealthCheck.build(
               type: Settings.worker_health_check_type,
               config: Settings.worker_health_check_config,
@@ -43,13 +45,16 @@ module Delayed
             )
             live_workers = checker.live_workers
 
-            Delayed::Job.running_jobs.each do |job|
-              # prefetched jobs have their own way of automatically unlocking themselves
-              next if job.locked_by.start_with?("prefetch:")
-
-              next if live_workers.include?(job.locked_by)
+            loop do
+              batch = Delayed::Job.running_jobs
+                                  .where("locked_at<?", horizon)
+                                  .where.not("locked_by LIKE 'prefetch:%'")
+                                  .where.not(locked_by: live_workers)
+                                  .limit(100)
+                                  .to_a
+              break if batch.empty?
 
-              begin
+              batch.each do |job|
                 Delayed::Job.transaction do
                   # double check that the job is still there. locked_by will immediately be reset
                   # to nil in this transaction by Job#reschedule
@@ -59,9 +64,9 @@ module Delayed
 
                   job.reschedule
                 end
-              rescue
-                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
+            rescue
+              ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
             end
           end
         end

data/spec/active_record_job_spec.rb

@@ -31,8 +31,8 @@ describe "Delayed::Backed::ActiveRecord::Job" do
       expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
     end
 
-    it "doesn't  allow a second worker to get exclusive access if failed to be processed by worker1 and
-        run_at time is now in future (due to backing off behaviour)" do
+    it "doesn't allow a second worker to get exclusive access if failed to be " \
+       "processed by worker1 and run_at time is now in future (due to backing off behaviour)" do
       @job.update(attempts: 1, run_at: 1.day.from_now)
       expect(@job_copy_for_worker2.send(:lock_exclusively!, "worker2")).to eq(false)
     end

data/spec/delayed/daemon_spec.rb

@@ -3,9 +3,10 @@
 require "spec_helper"
 
 RSpec.describe Delayed::Daemon do
+  subject { described_class.new(pid_folder) }
+
   let(:pid_folder) { "/test/pid/folder" }
   let(:pid) { 9999 }
-  let(:subject) { described_class.new(pid_folder) }
 
   before do
     allow(subject).to receive(:pid).and_return(pid)

data/spec/delayed/work_queue/in_process_spec.rb

@@ -11,7 +11,6 @@ RSpec.describe Delayed::WorkQueue::InProcess do
     Delayed::Worker.lifecycle.reset!
   end
 
-  let(:subject) { described_class.new }
   let(:worker_config) { { queue: "test", min_priority: 1, max_priority: 2 } }
   let(:args) { ["worker_name", worker_config] }
 

data/spec/delayed/work_queue/parent_process/client_spec.rb

@@ -3,7 +3,8 @@
 require "spec_helper"
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Client do
-  let(:subject) { described_class.new(addrinfo).tap(&:init) }
+  subject { described_class.new(addrinfo).tap(&:init) }
+
   let(:addrinfo) { double("Addrinfo") }
   let(:connection) { double("Socket") }
   let(:job) { Delayed::Job.new(locked_by: "worker_name") }

data/spec/delayed/work_queue/parent_process/server_spec.rb

@@ -15,14 +15,19 @@ class JobClass
 end
 
 RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
+  subject { described_class.new(listen_socket) }
+
   let(:parent) { Delayed::WorkQueue::ParentProcess.new }
-  let(:subject) { described_class.new(listen_socket) }
   let(:listen_socket) { Socket.unix_server_socket(parent.server_address) }
   let(:job) { JobClass.new }
   let(:worker_config) { { queue: "queue_name", min_priority: 1, max_priority: 2 } }
   let(:args) { ["worker_name", worker_config] }
   let(:job_args) { [["worker_name"], "queue_name", 1, 2, hash_including(prefetch: 4)] }
 
+  before do
+    Delayed::Worker.lifecycle.reset!
+  end
+
   before :all do
     Delayed.select_backend(Delayed::Backend::ActiveRecord::Job)
     Delayed::Settings.parent_process = {
@@ -36,6 +41,7 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
 
   after do
     File.unlink("/tmp/inst-jobs-test.sock") if File.exist?("/tmp/inst-jobs-test.sock")
+    Delayed::Worker.lifecycle.reset!
   end
 
   it "accepts new clients" do
@@ -204,4 +210,24 @@ RSpec.describe Delayed::WorkQueue::ParentProcess::Server do
     expect(Marshal.load(client)).to eq(job)
     expect(called).to eq(true)
   end
+
+  it "deletes the correct worker when transferring jobs" do
+    client1 = Socket.unix(subject.listen_socket.local_address.unix_path)
+    client2 = Socket.unix(subject.listen_socket.local_address.unix_path)
+    subject.run_once
+    subject.run_once
+
+    Marshal.dump(args, client1)
+    Marshal.dump(["worker_name2", worker_config], client2)
+    subject.run_once
+    subject.run_once
+
+    waiting_clients = subject.instance_variable_get(:@waiting_clients)
+    expect(waiting_clients.first.last.length).to eq 2
+
+    expect(Delayed::Job).to receive(:get_and_lock_next_available).and_return("worker_name" => job,
+                                                                             "worker_name2" => job)
+    subject.run_once
+    expect(waiting_clients.first.last).to be_empty
+  end
 end

data/spec/delayed/work_queue/parent_process_spec.rb

@@ -21,8 +21,6 @@ RSpec.describe Delayed::WorkQueue::ParentProcess do
     Delayed::Worker.lifecycle.reset!
   end
 
-  let(:subject) { described_class.new }
-
   describe "#initalize(config = Settings.parent_process)" do
     it "must expand a relative path to be within the Rails root" do
       queue = described_class.new("server_address" => "tmp/foo.sock")

data/spec/delayed/worker/health_check_spec.rb

@@ -51,7 +51,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
       end
     end
 
-    let(:initial_run_at) { Time.zone.now }
+    let(:initial_run_at) { 10.minutes.ago }
 
     before do
       klass.live_workers = %w[alive]
@@ -96,7 +96,13 @@ RSpec.describe Delayed::Worker::HealthCheck do
 
     it "ignores jobs that are re-locked after fetching from db" do
       Delayed::Job.where(id: @dead_job).update_all(locked_by: "someone_else")
-      allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
+      # we need to return @dead_job itself, which doesn't match the database
+      jobs_scope = double
+      allow(jobs_scope).to receive(:where).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:not).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:limit).and_return(jobs_scope)
+      allow(jobs_scope).to receive(:to_a).and_return([@dead_job], [])
+      allow(Delayed::Job).to receive(:running_jobs).and_return(jobs_scope)
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.locked_by).to eq "someone_else"
@@ -104,7 +110,7 @@ RSpec.describe Delayed::Worker::HealthCheck do
 
     it "ignores jobs that are prefetched" do
       Delayed::Job.where(id: @dead_job).update_all(locked_by: "prefetch:some_node")
-      allow(Delayed::Job).to receive(:running_jobs).and_return([@dead_job])
+      allow(Delayed::Job).to receive(:running_jobs).and_return(Delayed::Job.where(id: @dead_job.id))
       described_class.reschedule_abandoned_jobs
       @dead_job.reload
       expect(@dead_job.locked_by).to eq "prefetch:some_node"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 2.4.6
+  version: 2.4.10
 platform: ruby
 authors:
 - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-21 00:00:00.000000000 Z
+date: 2021-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -432,6 +432,7 @@ files:
 - db/migrate/20210809145804_add_n_strand_index.rb
 - db/migrate/20210812210128_add_singleton_column.rb
 - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
+- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb