inst-jobs 3.0.2 → 3.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 55850c386085b90bbb9df7ff954d40ed7f2035268860ec8b47dfc11f11ec7e51
-  data.tar.gz: 53065a1f09e8cd30271bc9b57ab58d3bcfff8cf1b9590568379427093de9cbe9
+  metadata.gz: 35f27ce4a29cf01568ecd625274894da1c2675e7d2e7da7d307af2200520140b
+  data.tar.gz: 75f5714d9269f601af9097c739fd6cb1948ef7899509c63ad3de2c7991800310
 SHA512:
-  metadata.gz: 8f7d579cddb80890c6158a1384f6828ae91272a864308528ad84e6dc47b3e4769062f91de7ebf9641f4b766c21c571a419385ee57bec9b6e5531f39be30a50a8
-  data.tar.gz: 887648362dec636b3e1f6938e66dd25b97f42a3e606c5d8466768d5b2ed5137ea50b67c68ce9ec817df86f117ef0660f6ddd3456c86e4871965f89704cc161f3
+  metadata.gz: 7eb2a9498036a106c8a02a7878094e940b7450bd60947c34bbb9246cc31f25950d16bb8b9c7eba6e7aa6d25bce2b1bbe11d603c9ca573bd87282225d0c36686f
+  data.tar.gz: 0fe9a7d4226558e665b8a961dd1fd88b5c7e9d076ed6927925a7a259ddce279077360da42746bfd1f064d1e8ad0f602c43b53fb92064996854e76980035f7cc0

data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionDelete < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/exe/inst_jobs

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require_relative "config/environment"
+require File.expand_path("config/environment")
 Delayed::CLI.new.run

data/lib/delayed/rails_reloader_plugin.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "delayed/plugin"
+
+module Delayed
+  class RailsReloaderPlugin < Plugin
+    callbacks do |lifecycle|
+      app = Rails.application
+      if app && !app.config.cache_classes
+        lifecycle.around(:perform) do |worker, job, &block|
+          reload = !app.config.reload_classes_only_on_change || app.reloaders.any?(&:updated?)
+
+          if reload
+            if defined?(ActiveSupport::Reloader)
+              Rails.application.reloader.reload!
+            else
+              ActionDispatch::Reloader.prepare!
+            end
+          end
+
+          begin
+            block.call(worker, job)
+          ensure
+            ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "3.0.2"
+  VERSION = "3.0.7"
 end

data/lib/delayed/worker.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "delayed/rails_reloader_plugin"
+
 module Delayed
   class TimeoutError < RuntimeError; end
 
@@ -70,27 +72,7 @@ module Delayed
 
       @signal_queue = []
 
-      app = Rails.application
-      if app && !app.config.cache_classes
-        Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
-          reload = app.config.reload_classes_only_on_change != true || app.reloaders.map(&:updated?).any?
-
-          if reload
-            if defined?(ActiveSupport::Reloader)
-              Rails.application.reloader.reload!
-            else
-              ActionDispatch::Reloader.prepare!
-            end
-          end
-
-          begin
-            block.call(worker, job)
-          ensure
-            ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
-          end
-        end
-      end
-
+      plugins << Delayed::RailsReloaderPlugin
       plugins.each(&:inject!)
     end
 

data/spec/delayed/worker_spec.rb

@@ -44,7 +44,7 @@ describe Delayed::Worker do
       expect(output_count).to eq(1)
     end
 
-    it "reloads" do
+    it "reloads Rails classes (never more than once)" do
       fake_application = double("Rails.application",
                                 config: double("Rails.application.config",
                                                cache_classes: false,
@@ -59,6 +59,11 @@ describe Delayed::Worker do
         expect(ActionDispatch::Reloader).to receive(:cleanup!).once
       end
       job = double(job_attrs)
+
+      # Create extra workers to make sure we don't reload multiple times
+      described_class.new(worker_config.dup)
+      described_class.new(worker_config.dup)
+
       subject.perform(job)
     end
   end

data/spec/shared/shared_backend.rb

@@ -389,7 +389,222 @@ shared_examples_for "a backend" do
         expect(job1.reload.handler).to include("ErrorJob")
       end
 
+      context "next_in_strand management - deadlocks and race conditions", non_transactional: true do
+        # The following unit tests are fairly slow and non-deterministic. It may be
+        # easier to make them fail quicker and more consistently by adding a random
+        # sleep into the appropriate trigger(s).
+
+        def loop_secs(val)
+          loop_start = Time.now.utc
+
+          loop do
+            break if Time.now.utc >= loop_start + val
+
+            yield
+          end
+        end
+
+        def loop_until_found(params)
+          found = false
+
+          loop_secs(10.seconds) do
+            if Delayed::Job.exists?(**params)
+              found = true
+              break
+            end
+          end
+
+          raise "timed out waiting for condition" unless found
+        end
+
+        def thread_body
+          yield
+        rescue
+          Thread.current.thread_variable_set(:fail, true)
+          raise
+        end
+
+        it "doesn't orphan the singleton when two are queued consecutively" do
+          # In order to reproduce this one efficiently, you'll probably want to add
+          # a sleep within delayed_jobs_before_insert_row_tr_fn.
+          # IF NEW.singleton IS NOT NULL THEN
+          #   ...
+          #   PERFORM pg_sleep(random() * 2);
+          # END IF;
+
+          threads = []
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                create_job(singleton: "singleton_job")
+                create_job(singleton: "singleton_job")
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                Delayed::Job.get_and_lock_next_available("w1")&.destroy
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "singleton_job", next_in_strand: true)
+              end
+            end
+          end
+
+          begin
+            loop_secs(60.seconds) do
+              if threads.any? { |x| x.thread_variable_get(:fail) }
+                raise "at least one job became orphaned or other error"
+              end
+            end
+          ensure
+            threads.each(&:kill)
+            threads.each(&:join)
+          end
+        end
+
+        it "doesn't deadlock when transitioning from strand_a to strand_b" do
+          # In order to reproduce this one efficiently, you'll probably want to add
+          # a sleep within delayed_jobs_after_delete_row_tr_fn.
+          # PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          # PERFORM pg_sleep(random() * 2);
+
+          threads = []
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                j1 = create_job(singleton: "myjobs", strand: "myjobs2", locked_by: "w1")
+                j2 = create_job(singleton: "myjobs", strand: "myjobs")
+
+                j1.delete
+                j2.delete
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                j1 = create_job(singleton: "myjobs2", strand: "myjobs", locked_by: "w1")
+                j2 = create_job(singleton: "myjobs2", strand: "myjobs2")
+
+                j1.delete
+                j2.delete
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "myjobs", next_in_strand: true)
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "myjobs2", next_in_strand: true)
+              end
+            end
+          end
+
+          begin
+            loop_secs(60.seconds) do
+              if threads.any? { |x| x.thread_variable_get(:fail) }
+                raise "at least one thread hit a deadlock or other error"
+              end
+            end
+          ensure
+            threads.each(&:kill)
+            threads.each(&:join)
+          end
+        end
+      end
+
       context "next_in_strand management" do
+        it "handles transitions correctly when going from stranded to not stranded" do
+          @job1 = create_job(singleton: "myjobs", strand: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job2 = create_job(singleton: "myjobs")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq false
+
+          @job1.destroy
+          expect(@job2.reload.next_in_strand).to eq true
+        end
+
+        it "handles transitions correctly when going from not stranded to stranded" do
+          @job1 = create_job(singleton: "myjobs2", strand: "myjobs")
+          @job2 = create_job(singleton: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job3 = create_job(singleton: "myjobs", strand: "myjobs2")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+
+          @job2.destroy
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq true
+        end
+
+        it "does not violate n_strand=1 constraints when going from not stranded to stranded" do
+          @job1 = create_job(singleton: "myjobs2", strand: "myjobs")
+          @job2 = create_job(singleton: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job3 = create_job(singleton: "myjobs", strand: "myjobs")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+
+          @job2.destroy
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+        end
+
+        it "handles transitions correctly when going from stranded to another strand" do
+          @job1 = create_job(singleton: "myjobs", strand: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job2 = create_job(singleton: "myjobs", strand: "myjobs2")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq false
+
+          @job1.destroy
+          expect(@job2.reload.next_in_strand).to eq true
+        end
+
+        it "does not violate n_strand=1 constraints when going from stranded to another strand" do
+          @job1 = create_job(singleton: "myjobs2", strand: "myjobs2")
+          @job2 = create_job(singleton: "myjobs", strand: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job3 = create_job(singleton: "myjobs", strand: "myjobs2")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+
+          @job2.destroy
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+        end
+
         it "creates first as true, and second as false, then transitions to second when deleted" do
           @job1 = create_job(singleton: "myjobs")
           Delayed::Job.get_and_lock_next_available("w1")

data/spec/spec_helper.rb

@@ -54,7 +54,8 @@ connection_config = {
   host: ENV["TEST_DB_HOST"].presence,
   encoding: "utf8",
   username: ENV["TEST_DB_USERNAME"],
-  database: ENV["TEST_DB_DATABASE"]
+  database: ENV["TEST_DB_DATABASE"],
+  min_messages: "notice"
 }
 
 def migrate(file)

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.0.7
 platform: ruby
 authors:
 - Cody Cutrer
 - Ethan Vizitei
 - Jacob Burroughs
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-11-08 00:00:00.000000000 Z
+date: 2022-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -424,7 +424,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.4.0
-description:
+description: 
 email:
 - cody@instructure.com
 - evizitei@instructure.com
@@ -466,6 +466,9 @@ files:
 - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
 - db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
 - db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
+- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
+- db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
+- db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
@@ -483,6 +486,7 @@ files:
 - lib/delayed/periodic.rb
 - lib/delayed/plugin.rb
 - lib/delayed/pool.rb
+- lib/delayed/rails_reloader_plugin.rb
 - lib/delayed/server.rb
 - lib/delayed/server/helpers.rb
 - lib/delayed/server/public/css/app.css
@@ -531,7 +535,7 @@ files:
 homepage: https://github.com/instructure/inst-jobs
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -546,32 +550,32 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.24
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files:
-- spec/sample_jobs.rb
-- spec/spec_helper.rb
-- spec/shared_jobs_specs.rb
-- spec/shared/performable_method.rb
-- spec/shared/testing.rb
-- spec/shared/delayed_batch.rb
-- spec/shared/worker.rb
-- spec/shared/delayed_method.rb
-- spec/shared/shared_backend.rb
-- spec/migrate/20140924140513_add_story_table.rb
-- spec/delayed/server_spec.rb
+- spec/active_record_job_spec.rb
 - spec/delayed/cli_spec.rb
 - spec/delayed/daemon_spec.rb
-- spec/delayed/worker_spec.rb
-- spec/delayed/periodic_spec.rb
 - spec/delayed/message_sending_spec.rb
+- spec/delayed/periodic_spec.rb
+- spec/delayed/server_spec.rb
 - spec/delayed/settings_spec.rb
 - spec/delayed/work_queue/in_process_spec.rb
-- spec/delayed/work_queue/parent_process_spec.rb
 - spec/delayed/work_queue/parent_process/client_spec.rb
 - spec/delayed/work_queue/parent_process/server_spec.rb
-- spec/delayed/worker/health_check_spec.rb
+- spec/delayed/work_queue/parent_process_spec.rb
 - spec/delayed/worker/consul_health_check_spec.rb
-- spec/active_record_job_spec.rb
+- spec/delayed/worker/health_check_spec.rb
+- spec/delayed/worker_spec.rb
+- spec/migrate/20140924140513_add_story_table.rb
+- spec/sample_jobs.rb
+- spec/shared/delayed_batch.rb
+- spec/shared/delayed_method.rb
+- spec/shared/performable_method.rb
+- spec/shared/shared_backend.rb
+- spec/shared/testing.rb
+- spec/shared/worker.rb
+- spec/shared_jobs_specs.rb
+- spec/spec_helper.rb