switchman-inst-jobs 3.1.0 → 3.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9466db757372227599ce8e15411f697c1955dc59bc94dba7143fcc0e4e18b47b
-  data.tar.gz: 0d5e44aecbbf1e9110dcb83e305ada21f98ae52c7b75f7ce052482dc648b4f39
+  metadata.gz: ac66ce458b7356068a5f576593f9692b5aa58d608584033a3aca941b31dc4a2d
+  data.tar.gz: c790f410b55117257f9b447987f68617b61222b10bf621385736b096affa25f8
 SHA512:
-  metadata.gz: 883bfa88ae5677c9b8929be849fd468d4b2861a2947c3c2bb626db12fe598c773f7e45706f4ff817558104e072d7c64aa5c73b7823c86fcb18361b06a5f5bd98
-  data.tar.gz: caa20756112d7667fbf00ab5fdfbea825f3515bd9593c5135a399b652e351f8722f49a05088db5347df03c29b900a6139d1dff37b975c723d28a8d2e35a198ae
+  metadata.gz: 5d21bcfec5e444aeaf4bf839c13ab40453ea7b27c780e9153aeea23d5692800ff0e36dac604c7b7dec7c7723aea6a1baba675748a543240a69b37a3486665110
+  data.tar.gz: ea0797fc7294cf1be7b641ad874e4560a69a6a7bc04228ee85f7c284f5fcc1288139b6824c72ae5db1c521b19fa3aec55d36c7831dc819046a8af7f2d623886b

data/db/migrate/20210809145804_add_n_strand_index.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class AddNStrandIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    add_index :delayed_jobs, %i[strand next_in_strand id],
+              name: 'n_strand_index',
+              where: 'strand IS NOT NULL',
+              algorithm: :concurrently
+  end
+end

data/db/migrate/20210812210128_add_singleton_column.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+class AddSingletonColumn < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    add_column :delayed_jobs, :singleton, :string, if_not_exists: true
+    add_column :failed_jobs, :singleton, :string, if_not_exists: true
+    # only one job can be queued in a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: 'singleton IS NOT NULL AND locked_by IS NULL',
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_not_running',
+              algorithm: :concurrently
+    # only one job can be running for a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: 'singleton IS NOT NULL AND locked_by IS NOT NULL',
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_running',
+              algorithm: :concurrently
+
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+          DECLARE
+            running_count integer;
+            should_lock boolean;
+            should_be_precise boolean;
+            update_query varchar;
+            skip_locked varchar;
+          BEGIN
+            IF OLD.strand IS NOT NULL THEN
+              should_lock := true;
+              should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+              IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) subquery_for_count);
+                should_lock := running_count < OLD.max_concurrent;
+              END IF;
+
+              IF should_lock THEN
+                PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+              END IF;
+
+              -- note that we don't really care if the row we're deleting has a singleton, or if it even
+              -- matches the row(s) we're going to update. we just need to make sure that whatever
+              -- singleton we grab isn't already running (which is a simple existence check, since
+              -- the unique indexes ensure there is at most one singleton running, and one queued)
+              update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+                SELECT id FROM delayed_jobs j2
+                  WHERE next_in_strand=false AND
+                    j2.strand=$1.strand AND
+                    (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
+                  ORDER BY j2.strand_order_override ASC, j2.id ASC
+                  LIMIT ';
+
+              IF should_be_precise THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) s);
+                IF running_count < OLD.max_concurrent THEN
+                  update_query := update_query || '($1.max_concurrent - $2)';
+                ELSE
+                  -- we have too many running already; just bail
+                  RETURN OLD;
+                END IF;
+              ELSE
+                update_query := update_query || '1';
+
+                -- n-strands don't require precise ordering; we can make this query more performant
+                IF OLD.max_concurrent > 1 THEN
+                  skip_locked := ' SKIP LOCKED';
+                END IF;
+              END IF;
+
+              update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+              EXECUTE update_query USING OLD, running_count;
+            ELSIF OLD.singleton IS NOT NULL THEN
+              UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
+            END IF;
+            RETURN OLD;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+          DECLARE
+            running_count integer;
+            should_lock boolean;
+            should_be_precise boolean;
+          BEGIN
+            IF OLD.strand IS NOT NULL THEN
+              should_lock := true;
+              should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+              IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) subquery_for_count);
+                should_lock := running_count < OLD.max_concurrent;
+              END IF;
+
+              IF should_lock THEN
+                PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+              END IF;
+
+              IF should_be_precise THEN
+                running_count := (SELECT COUNT(*) FROM (
+                  SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+                ) subquery_for_count);
+                IF running_count < OLD.max_concurrent THEN
+                  UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                    SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                  );
+                END IF;
+              ELSE
+                -- n-strands don't require precise ordering; we can make this query more performant
+                IF OLD.max_concurrent > 1 THEN
+                  UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+                ELSE
+                  UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                    (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                      j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
+                END IF;
+              END IF;
+            END IF;
+            RETURN OLD;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 AS one FROM delayed_jobs WHERE strand = NEW.strand LIMIT NEW.max_concurrent
+                ) subquery_for_count) = NEW.max_concurrent THEN
+                NEW.next_in_strand := 'f';
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+
+    connection.transaction do
+      reversible do |direction|
+        direction.up do
+          drop_triggers
+          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL OR NEW.singleton IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN ((OLD.strand IS NOT NULL OR OLD.singleton IS NOT NULL) AND OLD.next_in_strand=true) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
+        end
+        direction.down do
+          drop_triggers
+          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn()')}")
+        end
+      end
+    end
+  end
+
+  def drop_triggers
+    execute("DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name}")
+    execute("DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name}")
+  end
+end

data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class AddDeleteConflictingSingletonsBeforeUnlockTrigger < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')} () RETURNS trigger AS $$
+      BEGIN
+        IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
+          DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+    execute(<<~SQL)
+      CREATE TRIGGER delayed_jobs_before_unlock_delete_conflicting_singletons_row_tr BEFORE UPDATE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (
+        OLD.singleton IS NOT NULL AND
+        OLD.singleton=NEW.singleton AND
+        OLD.locked_by IS NOT NULL AND
+        NEW.locked_by IS NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')}();
+    SQL
+  end
+
+  def down
+    execute("DROP FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_tr_fn')}() CASCADE")
+  end
+end

data/db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class FixSingletonConditionInBeforeInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton;
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class UpdateConflictingSingletonFunctionToUseIndex < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')} () RETURNS trigger AS $$
+      BEGIN
+        DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton AND locked_by IS NULL;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_unlock_delete_conflicting_singletons_row_fn')} () RETURNS trigger AS $$
+      BEGIN
+        IF EXISTS (SELECT 1 FROM delayed_jobs j2 WHERE j2.singleton=OLD.singleton) THEN
+          DELETE FROM delayed_jobs WHERE id<>OLD.id AND singleton=OLD.singleton;
+        END IF;
+        RETURN NEW;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonIndex < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+class UpdateAfterDeleteTriggerForSingletonTransitionCases < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        ELSIF OLD.singleton IS NOT NULL THEN
+          UPDATE delayed_jobs SET next_in_strand = 't' WHERE singleton=OLD.singleton AND next_in_strand=false AND locked_by IS NULL;
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionDelete < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -18,7 +18,7 @@ module SwitchmanInstJobs
             enqueue_options = options.merge(
               current_shard: current_shard
             )
-            enqueue_job = -> { ::GuardRail.activate(:master) { super(object, **enqueue_options) } }
+            enqueue_job = -> { ::GuardRail.activate(:primary) { super(object, **enqueue_options) } }
 
             # Another dj shard must be currently manually activated, so just use that
             # In general this will only happen in unusual circumstances like tests
@@ -43,8 +43,7 @@ module SwitchmanInstJobs
           end
 
           def configured_shard_ids
-            (::Delayed::Settings.worker_config.try(:[], 'workers') || []).
-              map { |w| w['shard'] }.compact.uniq
+            ::SwitchmanInstJobs::Delayed::Settings.configured_shard_ids
           end
 
           def processes_locked_locally
@@ -74,7 +73,7 @@ module SwitchmanInstJobs
           self.shard_id = shard.id
           self.shard_id = nil if shard.is_a?(::Switchman::DefaultShard)
           # If jobs are held for a shard, enqueue new ones as held as well
-          return unless shard.jobs_held
+          return unless ::Switchman::Shard.columns_hash.key?('jobs_held') && shard.jobs_held
 
           self.locked_by = ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY
           self.locked_at = ::Delayed::Job.db_time_now
@@ -82,6 +81,8 @@ module SwitchmanInstJobs
         end
 
         def invoke_job
+          raise ShardNotFoundError, shard_id unless current_shard
+
           current_shard.activate { super }
         end
 

data/lib/switchman_inst_jobs/delayed/settings.rb

@@ -0,0 +1,9 @@
+module SwitchmanInstJobs
+  module Delayed
+    module Settings
+      def self.configured_shard_ids
+        (::Delayed::Settings.worker_config.try(:[], 'workers') || []).map { |w| w['shard'] }.compact.uniq
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/delayed/worker/health_check.rb

@@ -18,22 +18,23 @@ module SwitchmanInstJobs
             ::Delayed::Settings.worker_health_check_config['service_name'] = original_service_name
           end
 
-          def reschedule_abandoned_jobs(call_super: false)
-            shards = ::Switchman::Shard.delayed_jobs_shards.to_a
-            call_super = shards.first if shards.length == 1
-            unless call_super == false
-              call_super.activate(:delayed_jobs) do
-                return munge_service_name(call_super) { super() }
+          def reschedule_abandoned_jobs
+            shard_ids = ::SwitchmanInstJobs::Delayed::Settings.configured_shard_ids
+            shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
+            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
+              dj_shard = ::Switchman::Shard.current(:delayed_jobs)
+              dj_shard.activate do
+                munge_service_name(dj_shard) do
+                  # because this rescheduling process is running on every host, we need
+                  # to make sure that it's functioning for each shard the current
+                  # host is programmed to interact with, but ONLY for those shards.
+                  # reading the config lets us iterate over any shards this host should
+                  # work with and lets us pick the correct service name to identify which
+                  # hosts are currently alive and valid via the health checks
+                  super()
+                end
               end
             end
-
-            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs], exception: :ignore) do
-              shard = ::Switchman::Shard.current(:delayed_jobs)
-              singleton = <<~SINGLETON
-                periodic: Delayed::Worker::HealthCheck.reschedule_abandoned_jobs:#{shard.id}
-              SINGLETON
-              delay(singleton: singleton).reschedule_abandoned_jobs(call_super: shard)
-            end
           end
         end
       end

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -1,7 +1,3 @@
-# Just disabling all the rubocop metrics for this file for now,
-# as it is a direct port-in of existing code
-
-# rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
 require 'set'
 require 'parallel'
 
@@ -39,16 +35,12 @@ module SwitchmanInstJobs
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
         target_shards.each do |target_shard, shards|
-          ::Switchman::Shard.where(id: shards).update_all(delayed_jobs_shard_id: target_shard, block_stranded: true)
+          updates = { delayed_jobs_shard_id: target_shard, block_stranded: true }
+          updates[:updated_at] = Time.zone.now if ::Switchman::Shard.column_names.include?('updated_at')
+          ::Switchman::Shard.where(id: shards).update_all(updates)
         end
         clear_shard_cache
 
-        # Wait a little over the 60 second in-process shard cache clearing
-        # threshold to ensure that all new stranded jobs are now being
-        # enqueued with next_in_strand: false
-        Rails.logger.debug('Waiting for caches to clear')
-        sleep(65) unless @skip_cache_wait
-
         ::Switchman::Shard.clear_cache
         # rubocop:disable Style/CombinableLoops
         # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
@@ -59,11 +51,33 @@ module SwitchmanInstJobs
         source_shards.each do |s|
           ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
         end
+        ensure_unblock_stranded_for(shard_map.map(&:first))
         # rubocop:enable Style/CombinableLoops
       end
 
-      def clear_shard_cache
+      # if :migrate_strands ran on any shards that fell into scenario 1, then
+      # block_stranded never got flipped, so do that now.
+      def ensure_unblock_stranded_for(shards)
+        shards = ::Switchman::Shard.where(id: shards, block_stranded: true).to_a
+        return unless shards.any?
+
+        ::Switchman::Shard.where(id: shards).update_all(block_stranded: false)
+        clear_shard_cache
+
+        # shards is an array of shard objects that is now stale cause block_stranded has been updated.
+        shards.map(&:delayed_jobs_shard).uniq.each do |dj_shard|
+          unblock_strands(dj_shard)
+        end
+      end
+
+      def clear_shard_cache(debug_message = nil)
         ::Switchman.cache.clear
+        Rails.logger.debug("Waiting for caches to clear #{debug_message}")
+        # Wait a little over the 60 second in-process shard cache clearing
+        # threshold to ensure that all new stranded jobs are now being
+        # enqueued with next_in_strand: false
+        # @skip_cache_wait is for spec usage only
+        sleep(65) unless @skip_cache_wait
       end
 
       # This method expects that all relevant shards already have block_stranded: true
@@ -125,7 +139,7 @@ module SwitchmanInstJobs
                     update_all(next_in_strand: false)
                 end
 
-                # 4) is taken care of here, by leaveing next_in_strand alone and
+                # 4) is taken care of here, by leaving next_in_strand alone and
                 # it should execute on the new shard
                 batch_move_jobs(
                   target_shard: target_shard,
@@ -141,35 +155,32 @@ module SwitchmanInstJobs
             updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
               update_all(block_stranded: false)
             # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
-            unless updated.zero?
-              clear_shard_cache
-              # Wait a little over the 60 second in-process shard cache clearing
-              # threshold to ensure that all new stranded jobs are now being
-              # enqueued with next_in_strand: false
-              Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
-              # for spec usage only
-              sleep(65) unless @skip_cache_wait
-            end
+            clear_shard_cache("(#{source_shard.id} -> #{target_shard.id})") unless updated.zero?
+
             ::Switchman::Shard.clear_cache
             # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
             # but actually could have run and we just didn't know it because we didn't know if they had jobs
             # on the source shard
-            target_shard.activate(:delayed_jobs) do
-              loop do
-                # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
-                # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
-                # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
-                # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
-                # batches
-                break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
-                  where.not(strand: nil).
-                  where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
-                    ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
-                    where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
-                    where('dj2.strand = delayed_jobs.strand').arel.exists.not
-                  ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
-              end
-            end
+            unblock_strands(target_shard)
+          end
+        end
+      end
+
+      def unblock_strands(target_shard)
+        target_shard.activate(:delayed_jobs) do
+          loop do
+            # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
+            # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
+            # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
+            # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
+            # batches
+            break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
+              where.not(strand: nil).
+              where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
+                ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
+                where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
+                where('dj2.strand = delayed_jobs.strand').arel.exists.not
+              ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
           end
         end
       end
@@ -266,7 +277,10 @@ module SwitchmanInstJobs
         connection = ::Delayed::Job.connection
         quoted_keys = keys.map { |k| connection.quote_column_name(k) }.join(', ')
 
-        connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
+        connection.execute 'DROP TABLE IF EXISTS delayed_jobs_bulk_copy'
+        connection.execute "CREATE TEMPORARY TABLE delayed_jobs_bulk_copy
+          (LIKE #{::Delayed::Job.quoted_table_name} INCLUDING DEFAULTS)"
+        connection.execute "COPY delayed_jobs_bulk_copy (#{quoted_keys}) FROM STDIN"
         records.map do |record|
           connection.raw_connection.put_copy_data("#{keys.map { |k| quote_text(record[k]) }.join("\t")}\n")
         end
@@ -278,6 +292,9 @@ module SwitchmanInstJobs
         rescue StandardError => e
           raise connection.send(:translate_exception, e, 'COPY FROM STDIN')
         end
+        connection.execute "INSERT INTO #{::Delayed::Job.quoted_table_name} (#{quoted_keys})
+          SELECT #{quoted_keys} FROM delayed_jobs_bulk_copy
+          ON CONFLICT (singleton) WHERE singleton IS NOT NULL AND locked_by IS NULL DO NOTHING"
         result.cmd_tuples
       end
 
@@ -295,5 +312,3 @@ module SwitchmanInstJobs
     end
   end
 end
-
-# rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '3.1.0'.freeze
+  VERSION = '3.2.7'.freeze
 end

data/lib/switchman_inst_jobs.rb

@@ -14,10 +14,6 @@ module SwitchmanInstJobs
     ::Delayed::Backend::ActiveRecord::Job.prepend(
       Delayed::Backend::Base
     )
-    ::Delayed::Backend::Redis::Job.prepend(
-      Delayed::Backend::Base
-    )
-    ::Delayed::Backend::Redis::Job.column :shard_id, :integer
     ::Delayed::Pool.prepend Delayed::Pool
     ::Delayed::Worker.prepend Delayed::Worker
     ::Delayed::Worker::HealthCheck.prepend Delayed::Worker::HealthCheck
@@ -38,6 +34,7 @@ end
 
 require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
 require 'switchman_inst_jobs/active_record/migration'
+require 'switchman_inst_jobs/delayed/settings'
 require 'switchman_inst_jobs/delayed/backend/base'
 require 'switchman_inst_jobs/delayed/message_sending'
 require 'switchman_inst_jobs/delayed/pool'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.2.7
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-07 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 2.4.9
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 2.4.9
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +107,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: imperium
+  name: diplomat
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -296,6 +296,15 @@ files:
 - db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb
 - db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb
 - db/migrate/20200825011002_add_strand_order_override.rb
+- db/migrate/20210809145804_add_n_strand_index.rb
+- db/migrate/20210812210128_add_singleton_column.rb
+- db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
+- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
+- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
+- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
+- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
+- db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
+- db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
@@ -303,6 +312,7 @@ files:
 - lib/switchman_inst_jobs/delayed/backend/base.rb
 - lib/switchman_inst_jobs/delayed/message_sending.rb
 - lib/switchman_inst_jobs/delayed/pool.rb
+- lib/switchman_inst_jobs/delayed/settings.rb
 - lib/switchman_inst_jobs/delayed/worker.rb
 - lib/switchman_inst_jobs/delayed/worker/health_check.rb
 - lib/switchman_inst_jobs/engine.rb
@@ -318,7 +328,9 @@ files:
 homepage: https://github.com/instructure/switchman-inst-jobs
 licenses:
 - MIT
-metadata: {}
+metadata:
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -327,14 +339,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.