switchman-inst-jobs 4.0.0 → 4.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e76d31061b35e05fb85de802c868f5244c386f39490b236b7e2fa48160c1b4d
-  data.tar.gz: 81d2c12f67012e962b7d45157f5df7f045a3f4f0c45df2c9294661125524f234
+  metadata.gz: e6f9f61d51da46ec92185649cc434d9168740c491ade56179d13537d9552fa8b
+  data.tar.gz: 5c53dbe8087c254444f995ee738cb4164db03fc8a91ae823938a13b372e3eb0d
 SHA512:
-  metadata.gz: cb73f1c14a052801826a5f95cdfda52ecd73703cdc1b82b320a4b4ba69ba5ef2b4f77f01d41b1bec667df5b16a02529d8297f7de5726a392e65e086e0ad81570
-  data.tar.gz: 6b806f4e8e1e74e589090f75e54de9f12fe3e6ac5f97fc08296adc1c998055a082f7f815d0f7a6362d4745770e7791db100f640e6dadba5c8196ed0e940109b4
+  metadata.gz: 44c3483d276b435c93abc3547b458cde60739b9a4843465541065ae5d71eb8dcee9d2c4ebebf8b1fe4fc31fd6c7951f983d8af76c44a340e7e29ce811466778f
+  data.tar.gz: 0acdee9429002d28e9368bf29fce1b64fa492132f43b8ff884bca36f5b5aded965343a621c2e61b07de7bd18dcc059b5e11d1a8571a8319a0582d9dc8e1fa206

data/db/migrate/20210809145804_add_n_strand_index.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class AddNStrandIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    add_index :delayed_jobs, %i[strand next_in_strand id],
+              name: 'n_strand_index',
+              where: 'strand IS NOT NULL',
+              algorithm: :concurrently
+  end
+end

data/db/migrate/20210812210128_add_singleton_column.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+class AddSingletonColumn < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    add_column :delayed_jobs, :singleton, :string, if_not_exists: true
+    add_column :failed_jobs, :singleton, :string, if_not_exists: true
+    # only one job can be queued in a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: 'singleton IS NOT NULL AND locked_by IS NULL',
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_not_running',
+              algorithm: :concurrently
+    # only one job can be running for a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: 'singleton IS NOT NULL AND locked_by IS NOT NULL',
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_running',
+              algorithm: :concurrently
+
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+          DECLARE
+            running_count integer;
+            should_lock boolean;
+            should_be_precise boolean;
+            update_query varchar;
+            skip_locked varchar;
+          BEGIN
+            IF OLD.strand IS NOT NULL THEN
+              should_lock := true;
+              should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+              IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) subquery_for_count);
+                should_lock := running_count < OLD.max_concurrent;
+              END IF;
+
+              IF should_lock THEN
+                PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+              END IF;
+
+              -- note that we don't really care if the row we're deleting has a singleton, or if it even
+              -- matches the row(s) we're going to update. we just need to make sure that whatever
+              -- singleton we grab isn't already running (which is a simple existence check, since
+              -- the unique indexes ensure there is at most one singleton running, and one queued)
+              update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+                SELECT id FROM delayed_jobs j2
+                  WHERE next_in_strand=false AND
+                    j2.strand=$1.strand AND
+                    (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
+                  ORDER BY j2.strand_order_override ASC, j2.id ASC
+                  LIMIT ';
+
+              IF should_be_precise THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) s);
+                IF running_count < OLD.max_concurrent THEN
+                  update_query := update_query || '($1.max_concurrent - $2)';
+                ELSE
+                  -- we have too many running already; just bail
+                  RETURN OLD;
+                END IF;
+              ELSE
+                update_query := update_query || '1';
+
+                -- n-strands don't require precise ordering; we can make this query more performant
+                IF OLD.max_concurrent > 1 THEN
+                  skip_locked := ' SKIP LOCKED';
+                END IF;
+              END IF;
+
+              update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+              EXECUTE update_query USING OLD, running_count;
+            ELSIF OLD.singleton IS NOT NULL THEN
+              UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
+            END IF;
+            RETURN OLD;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+          DECLARE
+            running_count integer;
+            should_lock boolean;
+            should_be_precise boolean;
+          BEGIN
+            IF OLD.strand IS NOT NULL THEN
+              should_lock := true;
+              should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+              IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) subquery_for_count);
+                should_lock := running_count < OLD.max_concurrent;
+              END IF;
+
+              IF should_lock THEN
+                PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+              END IF;
+
+              IF should_be_precise THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) subquery_for_count);
+                IF running_count < OLD.max_concurrent THEN
+                  UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                    SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                  );
+                END IF;
+              ELSE
+                -- n-strands don't require precise ordering; we can make this query more performant
+                IF OLD.max_concurrent > 1 THEN
+                  UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+                ELSE
+                  UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                    (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                      j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
+                END IF;
+              END IF;
+            END IF;
+            RETURN OLD;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
+                ) subquery_for_count) = NEW.max_concurrent THEN
+                NEW.next_in_strand := 'f';
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+
+    connection.transaction do
+      reversible do |direction|
+        direction.up do
+          drop_triggers
+          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL OR NEW.singleton IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN ((OLD.strand IS NOT NULL OR OLD.singleton IS NOT NULL) AND OLD.next_in_strand=true) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
+        end
+        direction.down do
+          drop_triggers
+          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn()')}")
+        end
+      end
+    end
+  end
+
+  def drop_triggers
+    execute("DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name}")
+    execute("DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name}")
+  end
+end

data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class AddDeleteConflictingSingletonsBeforeUnlockTrigger < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')} () RETURNS trigger AS $$
+      BEGIN
+        IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
+          DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+    execute(<<~SQL)
+      CREATE TRIGGER delayed_jobs_before_unlock_delete_conflicting_singletons_row_tr BEFORE UPDATE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (
+        OLD.singleton IS NOT NULL AND
+        OLD.singleton=NEW.singleton AND
+        OLD.locked_by IS NOT NULL AND
+        NEW.locked_by IS NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')}();
+    SQL
+  end
+
+  def down
+    execute("DROP FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_tr_fn')}() CASCADE")
+  end
+end

data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class UpdateConflictingSingletonFunctionToUseIndex < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')} () RETURNS trigger AS $$
+      BEGIN
+        DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton AND locked_by IS NULL;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')} () RETURNS trigger AS $$
+      BEGIN
+        IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
+          DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonIndex < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -74,7 +74,7 @@ module SwitchmanInstJobs
           self.shard_id = shard.id
           self.shard_id = nil if shard.is_a?(::Switchman::DefaultShard)
           # If jobs are held for a shard, enqueue new ones as held as well
-          return unless shard.jobs_held
+          return unless ::Switchman::Shard.columns_hash.key?('jobs_held') && shard.jobs_held
 
           self.locked_by = ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY
           self.locked_at = ::Delayed::Job.db_time_now

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -1,7 +1,3 @@
-# Just disabling all the rubocop metrics for this file for now,
-# as it is a direct port-in of existing code
-
-# rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
 require 'set'
 require 'parallel'
 
@@ -76,7 +72,7 @@ module SwitchmanInstJobs
 
       def clear_shard_cache(debug_message = nil)
         ::Switchman.cache.clear
-        Rails.logger.debug("Waiting for caches to clear #{debug_message}")
+        Rails.logger.debug { "Waiting for caches to clear #{debug_message}" }
         # Wait a little over the 60 second in-process shard cache clearing
         # threshold to ensure that all new stranded jobs are now being
         # enqueued with next_in_strand: false
@@ -225,12 +221,14 @@ module SwitchmanInstJobs
           target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
 
           query = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
-            "WITH limited_jobs AS (#{target_jobs.to_sql}) " \
-            "UPDATE #{::Delayed::Job.quoted_table_name} " \
-            "SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)}, " \
-            "locked_at = #{::Delayed::Job.connection.quote(::Delayed::Job.db_time_now)} "\
-            "FROM limited_jobs WHERE limited_jobs.id=#{::Delayed::Job.quoted_table_name}.id " \
-            "RETURNING #{::Delayed::Job.quoted_table_name}.*"
+            <<~SQL
+              WITH limited_jobs AS (#{target_jobs.to_sql})
+              UPDATE #{::Delayed::Job.quoted_table_name}
+              SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)},
+              locked_at = #{::Delayed::Job.connection.quote(::Delayed::Job.db_time_now)}
+              FROM limited_jobs WHERE limited_jobs.id=#{::Delayed::Job.quoted_table_name}.id
+              RETURNING #{::Delayed::Job.quoted_table_name}.*
+            SQL
           end
 
           jobs = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
@@ -283,7 +281,10 @@ module SwitchmanInstJobs
         connection = ::Delayed::Job.connection
         quoted_keys = keys.map { |k| connection.quote_column_name(k) }.join(', ')
 
-        connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
+        connection.execute 'DROP TABLE IF EXISTS delayed_jobs_bulk_copy'
+        connection.execute "CREATE TEMPORARY TABLE delayed_jobs_bulk_copy
+          (LIKE #{::Delayed::Job.quoted_table_name} INCLUDING DEFAULTS)"
+        connection.execute "COPY delayed_jobs_bulk_copy (#{quoted_keys}) FROM STDIN"
         records.map do |record|
           connection.raw_connection.put_copy_data("#{keys.map { |k| quote_text(record[k]) }.join("\t")}\n")
         end
@@ -295,6 +296,9 @@ module SwitchmanInstJobs
         rescue StandardError => e
           raise connection.send(:translate_exception, e, 'COPY FROM STDIN')
         end
+        connection.execute "INSERT INTO #{::Delayed::Job.quoted_table_name} (#{quoted_keys})
+          SELECT #{quoted_keys} FROM delayed_jobs_bulk_copy
+          ON CONFLICT (singleton) WHERE singleton IS NOT NULL AND locked_by IS NULL DO NOTHING"
         result.cmd_tuples
       end
 
@@ -312,5 +316,3 @@ module SwitchmanInstJobs
     end
   end
 end
-
-# rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '4.0.0'.freeze
+  VERSION = '4.0.2'.freeze
 end

data/lib/switchman_inst_jobs.rb

@@ -17,10 +17,6 @@ module SwitchmanInstJobs
     ::Delayed::Backend::ActiveRecord::Job.prepend(
       Delayed::Backend::Base
     )
-    ::Delayed::Backend::Redis::Job.prepend(
-      Delayed::Backend::Base
-    )
-    ::Delayed::Backend::Redis::Job.column :shard_id, :integer
     ::Delayed::Pool.prepend Delayed::Pool
     ::Delayed::Worker.prepend Delayed::Worker
     ::Delayed::Worker::HealthCheck.prepend Delayed::Worker::HealthCheck

metadata

@@ -1,35 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.2
 platform: ruby
 authors:
 - Bryan Petty
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-24 00:00:00.000000000 Z
+date: 2021-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.1
+        version: 2.4.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.3.1
+        version: 2.4.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -238,6 +238,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -266,7 +294,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
-description:
+description: 
 email:
 - bpetty@instructure.com
 executables: []
@@ -303,6 +331,13 @@ files:
 - db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb
 - db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb
 - db/migrate/20200825011002_add_strand_order_override.rb
+- db/migrate/20210809145804_add_n_strand_index.rb
+- db/migrate/20210812210128_add_singleton_column.rb
+- db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
+- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
+- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
+- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
+- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb
@@ -327,8 +362,10 @@ files:
 homepage: https://github.com/instructure/switchman-inst-jobs
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+metadata:
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -343,8 +380,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.
 test_files: []