inst-jobs 3.0.1 → 3.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9d86b7b6161b9397c885a82d687f85d84e7a17db5261bfc5ef1bb7e9f25e2ac
-  data.tar.gz: 4a75f8ac73b5f0a0b19d8073bcbfe87f654bbbdb90e147ddf02066916653d4e0
+  metadata.gz: 245e01e85640f50351b2bca5ae7b7391c5373c0f8bc5128f1294d9e7566b5346
+  data.tar.gz: 38622bfe41a62682e198119a6d86e10210e06341d1a045f30cdabd6281600566
 SHA512:
-  metadata.gz: cd3f9df169146f34f7366e02830575bf27e2fd33e97d190bed34dd8be44e34431261111eef133368cd342668cabeb529af6e16c8985fcf15c99807ea7a07fbab
-  data.tar.gz: 3924f7b2f6f4eee37c4d92bfedfd683304dfd0c035a65f47ea9dae607c0e81b03c35860420f39a7dadf890d203c85fe24de68c90c80a48839f23b6193311b3e3
+  metadata.gz: ced7dd5a9cfe21b545d1ade7d05efabd53de6eff97589626dd38ce650b1da641d93811b48848339bed3057f3e873c77f6c6b4d5d3d29640a68694d40f20402b2
+  data.tar.gz: 82f0bfad222bef95dec0c154e6a649c184ba4f09c122db0a52e6f5d17f30ce836987def104d4b49fab4b293838e0ba37034d1cad3fdb4e2df2ebe6c7a0556ec3

data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonIndex < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION delayed_jobs_before_insert_row_tr_fn () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql;
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionDelete < ActiveRecord::Migration[6.0]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql;
+    SQL
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -309,7 +309,7 @@ module Delayed
           if Settings.silence_periodic_log
             ::ActiveRecord::Base.logger.silence(&block)
           else
-            block.call
+            yield
           end
         end
 
@@ -548,7 +548,7 @@ module Delayed
 
         def fail!
           attrs = attributes
-          attrs["original_job_id"] = attrs.delete("id")
+          attrs["original_job_id"] = attrs.delete("id") if Failed.columns_hash.key?("original_job_id")
           attrs["failed_at"] ||= self.class.db_time_now
           attrs.delete("next_in_strand")
           attrs.delete("max_concurrent")

data/lib/delayed/rails_reloader_plugin.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "delayed/plugin"
+
+module Delayed
+  class RailsReloaderPlugin < Plugin
+    callbacks do |lifecycle|
+      app = Rails.application
+      if app && !app.config.cache_classes
+        lifecycle.around(:perform) do |worker, job, &block|
+          reload = !app.config.reload_classes_only_on_change || app.reloaders.any?(&:updated?)
+
+          if reload
+            if defined?(ActiveSupport::Reloader)
+              Rails.application.reloader.reload!
+            else
+              ActionDispatch::Reloader.prepare!
+            end
+          end
+
+          begin
+            block.call(worker, job)
+          ensure
+            ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delayed/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Delayed
-  VERSION = "3.0.1"
+  VERSION = "3.0.6"
 end

data/lib/delayed/worker/process_helper.rb

@@ -5,10 +5,10 @@ module Delayed
     module ProcessHelper
       STAT_LINUX = "stat --format=%%Y /proc/$WORKER_PID"
       STAT_MAC = "ps -o lstart -p $WORKER_PID"
-      STAT = RUBY_PLATFORM =~ /darwin/ ? STAT_MAC : STAT_LINUX
+      STAT = RUBY_PLATFORM.include?("darwin") ? STAT_MAC : STAT_LINUX
       ALIVE_CHECK_LINUX = '[ -d "/proc/$WORKER_PID" ]'
       ALIVE_CHECK_MAC = "ps -p $WORKER_PID > /dev/null"
-      ALIVE_CHECK = RUBY_PLATFORM =~ /darwin/ ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
+      ALIVE_CHECK = RUBY_PLATFORM.include?("darwin") ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
       SCRIPT_TEMPLATE = <<-BASH
         WORKER_PID="%<pid>d" # an example, filled from ruby when the check is created
         ORIGINAL_MTIME="%<mtime>s" # an example, filled from ruby when the check is created
@@ -29,7 +29,7 @@ module Delayed
       BASH
 
       def self.mtime(pid)
-        if RUBY_PLATFORM =~ /darwin/
+        if RUBY_PLATFORM.include?("darwin")
           `ps -o lstart -p #{pid}`.sub(/\n$/, "").presence
         else
           File::Stat.new("/proc/#{pid}").mtime.to_i.to_s rescue nil

data/lib/delayed/worker.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "delayed/rails_reloader_plugin"
+
 module Delayed
   class TimeoutError < RuntimeError; end
 
@@ -70,27 +72,7 @@ module Delayed
 
       @signal_queue = []
 
-      app = Rails.application
-      if app && !app.config.cache_classes
-        Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
-          reload = app.config.reload_classes_only_on_change != true || app.reloaders.map(&:updated?).any?
-
-          if reload
-            if defined?(ActiveSupport::Reloader)
-              Rails.application.reloader.reload!
-            else
-              ActionDispatch::Reloader.prepare!
-            end
-          end
-
-          begin
-            block.call(worker, job)
-          ensure
-            ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
-          end
-        end
-      end
-
+      plugins << Delayed::RailsReloaderPlugin
       plugins.each(&:inject!)
     end
 

data/spec/active_record_job_spec.rb

@@ -219,14 +219,14 @@ describe "Delayed::Backed::ActiveRecord::Job" do
   end
 
   it "gets process ids from locked_by" do
-    3.times.map { Delayed::Job.create payload_object: SimpleJob.new }
+    Array.new(3) { Delayed::Job.create payload_object: SimpleJob.new }
     Delayed::Job.get_and_lock_next_available(["job42:2", "job42:9001"])
     expect(Delayed::Job.processes_locked_locally(name: "job42").sort).to eq [2, 9001]
     expect(Delayed::Job.processes_locked_locally(name: "jobnotme")).to be_empty
   end
 
   it "allows fetching multiple jobs at once" do
-    jobs = 3.times.map { Delayed::Job.create payload_object: SimpleJob.new }
+    jobs = Array.new(3) { Delayed::Job.create payload_object: SimpleJob.new }
     locked_jobs = Delayed::Job.get_and_lock_next_available(%w[worker1 worker2])
     expect(locked_jobs.length).to eq(2)
     expect(locked_jobs.keys).to eq(%w[worker1 worker2])
@@ -235,7 +235,7 @@ describe "Delayed::Backed::ActiveRecord::Job" do
   end
 
   it "allows fetching extra jobs" do
-    jobs = 5.times.map { Delayed::Job.create payload_object: SimpleJob.new }
+    jobs = Array.new(5) { Delayed::Job.create payload_object: SimpleJob.new }
     locked_jobs = Delayed::Job.get_and_lock_next_available(["worker1"],
                                                            prefetch: 2,
                                                            prefetch_owner: "work_queue")

data/spec/delayed/worker_spec.rb

@@ -44,7 +44,7 @@ describe Delayed::Worker do
       expect(output_count).to eq(1)
     end
 
-    it "reloads" do
+    it "reloads Rails classes (never more than once)" do
       fake_application = double("Rails.application",
                                 config: double("Rails.application.config",
                                                cache_classes: false,
@@ -59,6 +59,11 @@ describe Delayed::Worker do
         expect(ActionDispatch::Reloader).to receive(:cleanup!).once
       end
       job = double(job_attrs)
+
+      # Create extra workers to make sure we don't reload multiple times
+      described_class.new(worker_config.dup)
+      described_class.new(worker_config.dup)
+
       subject.perform(job)
     end
   end

data/spec/shared/shared_backend.rb

@@ -389,7 +389,222 @@ shared_examples_for "a backend" do
         expect(job1.reload.handler).to include("ErrorJob")
       end
 
+      context "next_in_strand management - deadlocks and race conditions", non_transactional: true do
+        # The following unit tests are fairly slow and non-deterministic. It may be
+        # easier to make them fail quicker and more consistently by adding a random
+        # sleep into the appropriate trigger(s).
+
+        def loop_secs(val)
+          loop_start = Time.now.utc
+
+          loop do
+            break if Time.now.utc >= loop_start + val
+
+            yield
+          end
+        end
+
+        def loop_until_found(params)
+          found = false
+
+          loop_secs(10.seconds) do
+            if Delayed::Job.exists?(**params)
+              found = true
+              break
+            end
+          end
+
+          raise "timed out waiting for condition" unless found
+        end
+
+        def thread_body
+          yield
+        rescue
+          Thread.current.thread_variable_set(:fail, true)
+          raise
+        end
+
+        it "doesn't orphan the singleton when two are queued consecutively" do
+          # In order to reproduce this one efficiently, you'll probably want to add
+          # a sleep within delayed_jobs_before_insert_row_tr_fn.
+          # IF NEW.singleton IS NOT NULL THEN
+          #   ...
+          #   PERFORM pg_sleep(random() * 2);
+          # END IF;
+
+          threads = []
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                create_job(singleton: "singleton_job")
+                create_job(singleton: "singleton_job")
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                Delayed::Job.get_and_lock_next_available("w1")&.destroy
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "singleton_job", next_in_strand: true)
+              end
+            end
+          end
+
+          begin
+            loop_secs(60.seconds) do
+              if threads.any? { |x| x.thread_variable_get(:fail) }
+                raise "at least one job became orphaned or other error"
+              end
+            end
+          ensure
+            threads.each(&:kill)
+            threads.each(&:join)
+          end
+        end
+
+        it "doesn't deadlock when transitioning from strand_a to strand_b" do
+          # In order to reproduce this one efficiently, you'll probably want to add
+          # a sleep within delayed_jobs_after_delete_row_tr_fn.
+          # PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          # PERFORM pg_sleep(random() * 2);
+
+          threads = []
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                j1 = create_job(singleton: "myjobs", strand: "myjobs2", locked_by: "w1")
+                j2 = create_job(singleton: "myjobs", strand: "myjobs")
+
+                j1.delete
+                j2.delete
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                j1 = create_job(singleton: "myjobs2", strand: "myjobs", locked_by: "w1")
+                j2 = create_job(singleton: "myjobs2", strand: "myjobs2")
+
+                j1.delete
+                j2.delete
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "myjobs", next_in_strand: true)
+              end
+            end
+          end
+
+          threads << Thread.new do
+            thread_body do
+              loop do
+                loop_until_found(singleton: "myjobs2", next_in_strand: true)
+              end
+            end
+          end
+
+          begin
+            loop_secs(60.seconds) do
+              if threads.any? { |x| x.thread_variable_get(:fail) }
+                raise "at least one thread hit a deadlock or other error"
+              end
+            end
+          ensure
+            threads.each(&:kill)
+            threads.each(&:join)
+          end
+        end
+      end
+
       context "next_in_strand management" do
+        it "handles transitions correctly when going from stranded to not stranded" do
+          @job1 = create_job(singleton: "myjobs", strand: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job2 = create_job(singleton: "myjobs")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq false
+
+          @job1.destroy
+          expect(@job2.reload.next_in_strand).to eq true
+        end
+
+        it "handles transitions correctly when going from not stranded to stranded" do
+          @job1 = create_job(singleton: "myjobs2", strand: "myjobs")
+          @job2 = create_job(singleton: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job3 = create_job(singleton: "myjobs", strand: "myjobs2")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+
+          @job2.destroy
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq true
+        end
+
+        it "does not violate n_strand=1 constraints when going from not stranded to stranded" do
+          @job1 = create_job(singleton: "myjobs2", strand: "myjobs")
+          @job2 = create_job(singleton: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job3 = create_job(singleton: "myjobs", strand: "myjobs")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+
+          @job2.destroy
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+        end
+
+        it "handles transitions correctly when going from stranded to another strand" do
+          @job1 = create_job(singleton: "myjobs", strand: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job2 = create_job(singleton: "myjobs", strand: "myjobs2")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq false
+
+          @job1.destroy
+          expect(@job2.reload.next_in_strand).to eq true
+        end
+
+        it "does not violate n_strand=1 constraints when going from stranded to another strand" do
+          @job1 = create_job(singleton: "myjobs2", strand: "myjobs2")
+          @job2 = create_job(singleton: "myjobs", strand: "myjobs")
+          Delayed::Job.get_and_lock_next_available("w1")
+          Delayed::Job.get_and_lock_next_available("w1")
+          @job3 = create_job(singleton: "myjobs", strand: "myjobs2")
+
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job2.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+
+          @job2.destroy
+          expect(@job1.reload.next_in_strand).to eq true
+          expect(@job3.reload.next_in_strand).to eq false
+        end
+
         it "creates first as true, and second as false, then transitions to second when deleted" do
           @job1 = create_job(singleton: "myjobs")
           Delayed::Job.get_and_lock_next_available("w1")

data/spec/spec_helper.rb

@@ -54,7 +54,8 @@ connection_config = {
   host: ENV["TEST_DB_HOST"].presence,
   encoding: "utf8",
   username: ENV["TEST_DB_USERNAME"],
-  database: ENV["TEST_DB_DATABASE"]
+  database: ENV["TEST_DB_DATABASE"],
+  min_messages: "notice"
 }
 
 def migrate(file)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.1
+  version: 3.0.6
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-10-20 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -298,6 +298,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.19'
+- !ruby/object:Gem::Dependency
+  name: rubocop-performance
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.12.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
@@ -451,6 +465,10 @@ files:
 - db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
 - db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
 - db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
+- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
+- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
+- db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
+- db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
@@ -468,6 +486,7 @@ files:
 - lib/delayed/periodic.rb
 - lib/delayed/plugin.rb
 - lib/delayed/pool.rb
+- lib/delayed/rails_reloader_plugin.rb
 - lib/delayed/server.rb
 - lib/delayed/server/helpers.rb
 - lib/delayed/server/public/css/app.css
@@ -531,32 +550,32 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files:
-- spec/sample_jobs.rb
-- spec/spec_helper.rb
-- spec/shared_jobs_specs.rb
-- spec/shared/performable_method.rb
-- spec/shared/testing.rb
-- spec/shared/delayed_batch.rb
-- spec/shared/worker.rb
-- spec/shared/delayed_method.rb
-- spec/shared/shared_backend.rb
-- spec/migrate/20140924140513_add_story_table.rb
-- spec/delayed/server_spec.rb
+- spec/active_record_job_spec.rb
 - spec/delayed/cli_spec.rb
 - spec/delayed/daemon_spec.rb
-- spec/delayed/worker_spec.rb
-- spec/delayed/periodic_spec.rb
 - spec/delayed/message_sending_spec.rb
+- spec/delayed/periodic_spec.rb
+- spec/delayed/server_spec.rb
 - spec/delayed/settings_spec.rb
 - spec/delayed/work_queue/in_process_spec.rb
-- spec/delayed/work_queue/parent_process_spec.rb
 - spec/delayed/work_queue/parent_process/client_spec.rb
 - spec/delayed/work_queue/parent_process/server_spec.rb
-- spec/delayed/worker/health_check_spec.rb
+- spec/delayed/work_queue/parent_process_spec.rb
 - spec/delayed/worker/consul_health_check_spec.rb
-- spec/active_record_job_spec.rb
+- spec/delayed/worker/health_check_spec.rb
+- spec/delayed/worker_spec.rb
+- spec/migrate/20140924140513_add_story_table.rb
+- spec/sample_jobs.rb
+- spec/shared/delayed_batch.rb
+- spec/shared/delayed_method.rb
+- spec/shared/performable_method.rb
+- spec/shared/shared_backend.rb
+- spec/shared/testing.rb
+- spec/shared/worker.rb
+- spec/shared_jobs_specs.rb
+- spec/spec_helper.rb