inst-jobs 3.0.5 → 3.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 11059ce779a0ff644edcae25335e23b7ad2a4448dfb48e8cbc95295f3e6c468c
-  data.tar.gz: 84dfa8a1185219823013e1363190b0469e397dd423105eb29a205fea1e7207fa
+  metadata.gz: 245e01e85640f50351b2bca5ae7b7391c5373c0f8bc5128f1294d9e7566b5346
+  data.tar.gz: 38622bfe41a62682e198119a6d86e10210e06341d1a045f30cdabd6281600566
 SHA512:
-  metadata.gz: 9e4c5291673edccd5760cd11d087102e0d8829d25793a6ca4f9f3255336b3abb1ce2bcf7426a79a76d0f373b44ec2b0152c395974f78a4552942f7a858d9d499
-  data.tar.gz: 77aea9b18c3492e2b98f26a23c0591c8da9a31a34c0736868f00d54b3eadff77f7b2bbeb00b91b0819f93bffcefb7a26d1f2c829ebd82bc6b58a5c5a7bb01b16
+  metadata.gz: ced7dd5a9cfe21b545d1ade7d05efabd53de6eff97589626dd38ce650b1da641d93811b48848339bed3057f3e873c77f6c6b4d5d3d29640a68694d40f20402b2
+  data.tar.gz: 82f0bfad222bef95dec0c154e6a649c184ba4f09c122db0a52e6f5d17f30ce836987def104d4b49fab4b293838e0ba37034d1cad3fdb4e2df2ebe6c7a0556ec3

data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionDelete < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "3.0.5"
+  VERSION = "3.0.6"
 end

data/spec/shared/shared_backend.rb

@@ -389,14 +389,10 @@ shared_examples_for "a backend" do
         expect(job1.reload.handler).to include("ErrorJob")
       end
 
-      context "next_in_strand management - deadlocks", non_transactional: true do
+      context "next_in_strand management - deadlocks and race conditions", non_transactional: true do
         # The following unit tests are fairly slow and non-deterministic. It may be
         # easier to make them fail quicker and more consistently by adding a random
         # sleep into the appropriate trigger(s).
-        #
-        # Example:
-        #   PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
-        #   PERFORM pg_sleep(random() * 2);
 
         def loop_secs(val)
           loop_start = Time.now.utc
@@ -408,46 +404,119 @@ shared_examples_for "a backend" do
           end
         end
 
-        it "doesn't deadlock when transitioning from strand_a to strand_b" do
+        def loop_until_found(params)
+          found = false
+
+          loop_secs(10.seconds) do
+            if Delayed::Job.exists?(**params)
+              found = true
+              break
+            end
+          end
+
+          raise "timed out waiting for condition" unless found
+        end
+
+        def thread_body
+          yield
+        rescue
+          Thread.current.thread_variable_set(:fail, true)
+          raise
+        end
+
+        it "doesn't orphan the singleton when two are queued consecutively" do
+          # In order to reproduce this one efficiently, you'll probably want to add
+          # a sleep within delayed_jobs_before_insert_row_tr_fn.
+          # IF NEW.singleton IS NOT NULL THEN
+          #   ...
+          #   PERFORM pg_sleep(random() * 2);
+          # END IF;
+
           threads = []
 
-          def thread_body(j1_params, j2_params)
-            loop do
-              j1 = create_job(**j1_params)
-              j2 = create_job(**j2_params)
+          threads << Thread.new do
+            thread_body do
+              loop do
+                create_job(singleton: "singleton_job")
+                create_job(singleton: "singleton_job")
+              end
+            end
+          end
 
-              expect(j1.reload.next_in_strand).to eq(true)
-              expect(j2.reload.next_in_strand).to eq(false)
+          threads << Thread.new do
+            thread_body do
+              loop do
+                Delayed::Job.get_and_lock_next_available("w1")&.destroy
+              end
+            end
+          end
 
-              j1.delete
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "singleton_job", next_in_strand: true)
+              end
+            end
+          end
 
-              # In case we couldn't acquire a lock, we actually need to wait for
-              # the other thread to set this to true.
-              loop_secs(10.seconds) do
-                break if j2.reload.next_in_strand
+          begin
+            loop_secs(60.seconds) do
+              if threads.any? { |x| x.thread_variable_get(:fail) }
+                raise "at least one job became orphaned or other error"
               end
+            end
+          ensure
+            threads.each(&:kill)
+            threads.each(&:join)
+          end
+        end
+
+        it "doesn't deadlock when transitioning from strand_a to strand_b" do
+          # In order to reproduce this one efficiently, you'll probably want to add
+          # a sleep within delayed_jobs_after_delete_row_tr_fn.
+          # PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          # PERFORM pg_sleep(random() * 2);
 
-              expect(j2.reload.next_in_strand).to eq(true)
+          threads = []
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                j1 = create_job(singleton: "myjobs", strand: "myjobs2", locked_by: "w1")
+                j2 = create_job(singleton: "myjobs", strand: "myjobs")
 
-              j2.delete
+                j1.delete
+                j2.delete
+              end
             end
-          rescue
-            Thread.current.thread_variable_set(:fail, true)
-            raise
           end
 
           threads << Thread.new do
-            thread_body(
-              { singleton: "myjobs", strand: "myjobs2", locked_by: "w1" },
-              { singleton: "myjobs", strand: "myjobs" }
-            )
+            thread_body do
+              loop do
+                j1 = create_job(singleton: "myjobs2", strand: "myjobs", locked_by: "w1")
+                j2 = create_job(singleton: "myjobs2", strand: "myjobs2")
+
+                j1.delete
+                j2.delete
+              end
+            end
           end
 
           threads << Thread.new do
-            thread_body(
-              { singleton: "myjobs2", strand: "myjobs", locked_by: "w1" },
-              { singleton: "myjobs2", strand: "myjobs2" }
-            )
+            thread_body do
+              loop do
+                loop_until_found(singleton: "myjobs", next_in_strand: true)
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "myjobs2", next_in_strand: true)
+              end
+            end
           end
 
           begin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.5
+  version: 3.0.6
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-12-09 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -467,6 +467,8 @@ files:
 - db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
 - db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
 - db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
+- db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
+- db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb