switchman-inst-jobs 3.2.6 → 4.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fc596dd957a48e77a57106b08a3474fa39b2127f4c2a929fd363b3208e2a2e59
-  data.tar.gz: 39157783192cf59506d320ec36e7426e2239dd6de248a32c60cfcbf8eedf667e
+  metadata.gz: f07c44f5a58d897c8dbcc46a40906cde8c6a99b39723286a405669cf159019e4
+  data.tar.gz: cf730bfc3d3ad6d2c7da7712c3c097322421820b60f2d6af025ef610f0f3aa09
 SHA512:
-  metadata.gz: 5b256e4ac3fe9a779faada8b57df4725f1a227af4fddff925277ca5b68fa476ec5fd51bbbf96ff996e393089d162f41c043a3225587a79e4df4096d9ecdc4168
-  data.tar.gz: 5b43c7df21c598d1e9838e62e982ebe1ec31ab7a3daafa7b48bfafb882f3b716ced376eca1b2b80c27895e58a04bbd684fcfbed437712efcb20aa5d5ef780d55
+  metadata.gz: 9189978ce61d257fb25bbf57c23a36998437b3c4aa6adb93798af07ea878caf244a1514c5711c5190b39836169607e869f65b89ed71cc08d44fabfd9d6fe0e8e
+  data.tar.gz: d4a76e521ee4ba38bb2172a1ffabcc4b139a204d24690a2662ca7e8587d367ba88d98db5912c716421c96940f309296f6b35e6b292025326d61e32e03217d001

data/db/migrate/20101216224513_create_delayed_jobs.rb

@@ -1,6 +1,6 @@
 class CreateDelayedJobs < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20110208031356_add_delayed_jobs_tag.rb

@@ -1,6 +1,6 @@
 class AddDelayedJobsTag < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb

@@ -1,6 +1,6 @@
 class AddDelayedJobsMaxAttempts < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20110516225834_add_delayed_jobs_strand.rb

@@ -1,6 +1,6 @@
 class AddDelayedJobsStrand < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb

@@ -1,6 +1,6 @@
 class CleanupDelayedJobsIndexes < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20110610213249_optimize_delayed_jobs.rb

@@ -1,6 +1,6 @@
 class OptimizeDelayedJobs < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb

@@ -1,6 +1,6 @@
 class AddDelayedJobsNextInStrand < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb

@@ -1,6 +1,6 @@
 class DelayedJobsDeleteTriggerLockForUpdate < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb

@@ -1,6 +1,6 @@
 class DropPsqlJobsPopFn < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb

@@ -1,6 +1,6 @@
 class DelayedJobsUseAdvisoryLocks < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20120607181141_index_jobs_on_locked_by.rb

@@ -1,8 +1,8 @@
 class IndexJobsOnLockedBy < ActiveRecord::Migration[4.2]
-  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+  disable_ddl_transaction!
 
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20120608191051_add_jobs_run_at_index.rb

@@ -1,8 +1,8 @@
 class AddJobsRunAtIndex < ActiveRecord::Migration[4.2]
-  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+  disable_ddl_transaction!
 
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb

@@ -1,6 +1,6 @@
 class ChangeDelayedJobsHandlerToText < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb

@@ -1,6 +1,6 @@
 class AddFailedJobsOriginalJobId < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb

@@ -1,11 +1,11 @@
 class CopyFailedJobsOriginalId < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up
     # this is a smaller, less frequently accessed table, so we just update all at once
-    Delayed::Backend::ActiveRecord::Job::Failed.where('original_job_id is null').update_all('original_job_id = original_id')
+    Delayed::Backend::ActiveRecord::Job::Failed.where(original_job_id: nil).update_all('original_job_id = original_id')
   end
 
   def down; end

data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb

@@ -1,6 +1,6 @@
 class DropFailedJobsOriginalId < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20140512213941_add_source_to_jobs.rb

@@ -1,6 +1,6 @@
 class AddSourceToJobs < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb

@@ -1,6 +1,6 @@
 class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20151123210429_add_expires_at_to_jobs.rb

@@ -1,6 +1,6 @@
 class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20151210162949_improve_max_concurrent.rb

@@ -1,6 +1,6 @@
 class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb

@@ -1,6 +1,6 @@
 class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb

@@ -2,7 +2,7 @@ class AddShardIdToDelayedJobs < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
 
   def connection
-    Delayed::Backend::ActiveRecord::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up
@@ -11,19 +11,9 @@ class AddShardIdToDelayedJobs < ActiveRecord::Migration[4.2]
 
     add_column :failed_jobs, :shard_id, :integer, limit: 8
     add_index :failed_jobs, :shard_id, algorithm: :concurrently
-
-    add_column :switchman_shards, :delayed_jobs_shard_id, :integer, limit: 8
-    add_foreign_key(
-      :switchman_shards,
-      :switchman_shards,
-      column: :delayed_jobs_shard_id
-    )
   end
 
   def down
-    remove_foreign_key :switchman_shards, column: :delayed_jobs_shard_id
-    remove_column :switchman_shards, :delayed_jobs_shard_id
-
     remove_index :failed_jobs, :shard_id
     remove_column :failed_jobs, :shard_id
 

data/db/migrate/20170308045401_add_delayed_jobs_shard_id_to_switchman_shards.rb

@@ -0,0 +1,5 @@
+class AddDelayedJobsShardIdToSwitchmanShards < ActiveRecord::Migration[5.2]
+  def change
+    add_reference :switchman_shards, :delayed_jobs_shard, foreign_key: { to_table: :switchman_shards }, index: false, if_not_exists: true
+  end
+end

data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb

@@ -1,6 +1,6 @@
 class SpeedUpMaxConcurrentTriggers < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20190726154743_make_critical_columns_not_null.rb

@@ -1,6 +1,6 @@
 class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb

@@ -1,8 +1,8 @@
 class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
-  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+  disable_ddl_transaction!
 
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb

@@ -1,6 +1,6 @@
 class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20200825011002_add_strand_order_override.rb

@@ -1,8 +1,8 @@
 class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
-  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+  disable_ddl_transaction!
 
   def connection
-    Delayed::Job.connection
+    Delayed::Backend::ActiveRecord::AbstractJob.connection
   end
 
   def up

data/db/migrate/20211220112800_fix_singleton_race_condition_insert.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionInsert < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20211220113000_fix_singleton_race_condition_delete.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+class FixSingletonRaceConditionDelete < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb

@@ -0,0 +1,15 @@
+module SwitchmanInstJobs
+  module ActiveRecord
+    module ConnectionAdapters
+      module ConnectionPool
+        def shard
+          if connection_klass == ::Delayed::Backend::ActiveRecord::AbstractJob
+            return shard_stack.last || ::Switchman::Shard.current(::ActiveRecord::Base).delayed_jobs_shard
+          end
+
+          super
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -24,19 +24,18 @@ module SwitchmanInstJobs
             # In general this will only happen in unusual circumstances like tests
             # also if migrations are running, always use the current shard's job shard
             if ::ActiveRecord::Migration.open_migrations.zero? &&
-               current_shard.delayed_jobs_shard != ::Switchman::Shard.current(:delayed_jobs)
+               current_shard.delayed_jobs_shard !=
+               ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
               enqueue_job.call
             else
-              ::Switchman::Shard.default.activate do
-                current_shard = ::Switchman::Shard.lookup(current_shard.id)
-              end
+              current_shard = ::Switchman::Shard.lookup(current_shard.id)
               current_job_shard = current_shard.delayed_jobs_shard
 
               if (options[:singleton] || options[:strand]) && current_shard.block_stranded
                 enqueue_options[:next_in_strand] = false
               end
 
-              current_job_shard.activate(:delayed_jobs) do
+              current_job_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
                 enqueue_job.call
               end
             end
@@ -50,7 +49,7 @@ module SwitchmanInstJobs
             shard_ids = configured_shard_ids
             if shard_ids.any?
               shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
-              ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
+              ::Switchman::Shard.with_each_shard(shards, [::Delayed::Backend::ActiveRecord::AbstractJob]) do
                 super
               end
             else
@@ -61,7 +60,9 @@ module SwitchmanInstJobs
 
         def self.prepended(base)
           base.singleton_class.prepend(ClassMethods)
-          base.shard_category = :delayed_jobs if base.name == 'Delayed::Backend::ActiveRecord::Job'
+          return unless base.name == 'Delayed::Backend::ActiveRecord::Job'
+
+          ::Delayed::Backend::ActiveRecord::AbstractJob.sharded_model
         end
 
         def current_shard
@@ -90,7 +91,7 @@ module SwitchmanInstJobs
           raise ShardNotFoundError, shard_id unless current_shard
 
           current_shard.activate { super }
-        rescue ::Switchman::ConnectionError, PG::ConnectionBad, PG::UndefinedTable
+        rescue PG::ConnectionBad, PG::UndefinedTable
           # likely a missing shard with a stale cache
           current_shard.send(:clear_cache)
           ::Switchman::Shard.clear_cache

data/lib/switchman_inst_jobs/delayed/pool.rb

@@ -20,7 +20,7 @@ module SwitchmanInstJobs
           shard_ids = @config[:workers].pluck(:shard).uniq
           shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
         end
-        ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
+        ::Switchman::Shard.with_each_shard(shards, [::Delayed::Backend::ActiveRecord::AbstractJob]) do
           super
         end
       end

data/lib/switchman_inst_jobs/delayed/worker/health_check.rb

@@ -21,18 +21,16 @@ module SwitchmanInstJobs
           def reschedule_abandoned_jobs
             shard_ids = ::SwitchmanInstJobs::Delayed::Settings.configured_shard_ids
             shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
-            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
-              dj_shard = ::Switchman::Shard.current(:delayed_jobs)
-              dj_shard.activate do
-                munge_service_name(dj_shard) do
-                  # because this rescheduling process is running on every host, we need
-                  # to make sure that it's functioning for each shard the current
-                  # host is programmed to interact with, but ONLY for those shards.
-                  # reading the config lets us iterate over any shards this host should
-                  # work with and lets us pick the correct service name to identify which
-                  # hosts are currently alive and valid via the health checks
-                  super()
-                end
+            ::Switchman::Shard.with_each_shard(shards,
+                                               [::ActiveRecord::Base, ::Delayed::Backend::ActiveRecord::AbstractJob]) do
+              munge_service_name(::Switchman::Shard.current) do
+                # because this rescheduling process is running on every host, we need
+                # to make sure that it's functioning for each shard the current
+                # host is programmed to interact with, but ONLY for those shards.
+                # reading the config lets us iterate over any shards this host should
+                # work with and lets us pick the correct service name to identify which
+                # hosts are currently alive and valid via the health checks
+                super()
               end
             end
           end

data/lib/switchman_inst_jobs/delayed/worker.rb

@@ -16,14 +16,14 @@ module SwitchmanInstJobs
       end
 
       def start
-        shard.activate(:delayed_jobs) { super }
+        shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) { super }
       end
 
       # Worker#run is usually only called from Worker#start, but if the worker
       # is called directly from the console, we want to make sure it still gets
       # the right shard activated.
       def run
-        shard.activate(:delayed_jobs) { super }
+        shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) { super }
       end
 
       def shard

data/lib/switchman_inst_jobs/engine.rb

@@ -11,7 +11,9 @@ module SwitchmanInstJobs
 
       ::Delayed::Worker.lifecycle.around(:work_queue_pop) do |worker, config, &block|
         if config[:shard]
-          ::Switchman::Shard.lookup(config[:shard]).activate(:delayed_jobs) { block.call(worker, config) }
+          ::Switchman::Shard.lookup(config[:shard]).activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+            block.call(worker, config)
+          end
         else
           block.call(worker, config)
         end
@@ -23,8 +25,8 @@ module SwitchmanInstJobs
           ::Switchman::Shard.clear_cache
           ::Switchman::Shard.default.activate do
             current_job_shard = ::Switchman::Shard.lookup(job.shard_id).delayed_jobs_shard
-            if current_job_shard != ::Switchman::Shard.current(:delayed_jobs)
-              current_job_shard.activate(:delayed_jobs) do
+            if current_job_shard != ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
+              current_job_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
                 j = ::Delayed::Job.where(strand: job.strand).next_in_strand_order.first
                 j.update_column(:next_in_strand, true) if j && !j.next_in_strand
               end
@@ -43,7 +45,7 @@ module SwitchmanInstJobs
     end
 
     config.after_initialize do
-      ::Switchman::Shard.default.delayed_jobs_shard.activate!(:delayed_jobs)
+      ::Switchman::Shard.default.delayed_jobs_shard.activate!(::Delayed::Backend::ActiveRecord::AbstractJob)
     end
   end
 end

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -13,10 +13,10 @@ module SwitchmanInstJobs
         return yield if shards.empty?
 
         shard = shards.pop
-        current_shard = ::Switchman::Shard.current(:delayed_jobs)
-        shard.activate(:delayed_jobs) do
+        current_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
+        shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
           ::Delayed::Job.transaction do
-            current_shard.activate(:delayed_jobs) do
+            current_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
               transaction_on(shards, &block)
             end
           end
@@ -45,11 +45,11 @@ module SwitchmanInstJobs
         # rubocop:disable Style/CombinableLoops
         # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
         source_shards.each do |s|
-          ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_strands }
+          ::Switchman::Shard.lookup(s).activate(::Delayed::Backend::ActiveRecord::AbstractJob) { migrate_strands }
         end
 
         source_shards.each do |s|
-          ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
+          ::Switchman::Shard.lookup(s).activate(::Delayed::Backend::ActiveRecord::AbstractJob) { migrate_everything }
         end
         ensure_unblock_stranded_for(shard_map.map(&:first))
         # rubocop:enable Style/CombinableLoops
@@ -72,7 +72,7 @@ module SwitchmanInstJobs
 
       def clear_shard_cache(debug_message = nil)
         ::Switchman.cache.clear
-        Rails.logger.debug("Waiting for caches to clear #{debug_message}")
+        Rails.logger.debug { "Waiting for caches to clear #{debug_message}" }
         # Wait a little over the 60 second in-process shard cache clearing
         # threshold to ensure that all new stranded jobs are now being
         # enqueued with next_in_strand: false
@@ -99,7 +99,7 @@ module SwitchmanInstJobs
         # 4) no running job, jobs moved: set next_in_strand=true on the first of
         #    those (= do nothing since it should already be true)
 
-        source_shard = ::Switchman::Shard.current(:delayed_jobs)
+        source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
         strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
         shard_map = build_shard_map(strand_scope, source_shard)
         shard_map.each do |(target_shard, source_shard_ids)|
@@ -108,7 +108,7 @@ module SwitchmanInstJobs
           # 1) is taken care of because it should not show up here in strands
           strands = shard_scope.distinct.order(:strand).pluck(:strand)
 
-          target_shard.activate(:delayed_jobs) do
+          target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
             strands.each do |strand|
               transaction_on([source_shard, target_shard]) do
                 this_strand_scope = shard_scope.where(strand: strand)
@@ -167,7 +167,7 @@ module SwitchmanInstJobs
       end
 
       def unblock_strands(target_shard)
-        target_shard.activate(:delayed_jobs) do
+        target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
           loop do
             # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
             # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
@@ -186,8 +186,8 @@ module SwitchmanInstJobs
       end
 
       def migrate_everything
-        source_shard = ::Switchman::Shard.current(:delayed_jobs)
-        scope = ::Delayed::Job.shard(source_shard).where('strand IS NULL')
+        source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
+        scope = ::Delayed::Job.shard(source_shard).where(strand: nil)
 
         shard_map = build_shard_map(scope, source_shard)
         shard_map.each do |(target_shard, source_shard_ids)|
@@ -220,16 +220,20 @@ module SwitchmanInstJobs
           # Adapted from get_and_lock_next_available in delayed/backend/active_record.rb
           target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
 
-          query = source_shard.activate(:delayed_jobs) do
-            "WITH limited_jobs AS (#{target_jobs.to_sql}) " \
-            "UPDATE #{::Delayed::Job.quoted_table_name} " \
-            "SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)}, " \
-            "locked_at = #{::Delayed::Job.connection.quote(::Delayed::Job.db_time_now)} "\
-            "FROM limited_jobs WHERE limited_jobs.id=#{::Delayed::Job.quoted_table_name}.id " \
-            "RETURNING #{::Delayed::Job.quoted_table_name}.*"
+          query = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+            <<~SQL
+              WITH limited_jobs AS (#{target_jobs.to_sql})
+              UPDATE #{::Delayed::Job.quoted_table_name}
+              SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)},
+              locked_at = #{::Delayed::Job.connection.quote(::Delayed::Job.db_time_now)}
+              FROM limited_jobs WHERE limited_jobs.id=#{::Delayed::Job.quoted_table_name}.id
+              RETURNING #{::Delayed::Job.quoted_table_name}.*
+            SQL
           end
 
-          jobs = source_shard.activate(:delayed_jobs) { ::Delayed::Job.find_by_sql(query) }
+          jobs = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+            ::Delayed::Job.find_by_sql(query)
+          end
           new_jobs = jobs.map do |job|
             new_job = job.dup
             new_job.shard = target_shard
@@ -247,10 +251,10 @@ module SwitchmanInstJobs
             new_job
           end
           transaction_on([source_shard, target_shard]) do
-            target_shard.activate(:delayed_jobs) do
+            target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
               bulk_insert_jobs(new_jobs)
             end
-            source_shard.activate(:delayed_jobs) do
+            source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
               ::Delayed::Job.delete(jobs)
             end
           end

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -22,12 +22,12 @@ module SwitchmanInstJobs
       def hold_jobs!(wait: false)
         self.jobs_held = true
         save! if changed?
-        delayed_jobs_shard.activate(:delayed_jobs) do
+        delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
           lock_jobs_for_hold
         end
         return unless wait
 
-        delayed_jobs_shard.activate(:delayed_jobs) do
+        delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
           while ::Delayed::Job.where(shard_id: id).
               where.not(locked_at: nil).
               where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
@@ -47,7 +47,7 @@ module SwitchmanInstJobs
           Rails.logger.debug('Waiting for caches to clear')
           sleep(65)
         end
-        delayed_jobs_shard.activate(:delayed_jobs) do
+        delayed_jobs_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
           ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id).
             in_batches(of: 10_000).
             update_all(
@@ -75,22 +75,14 @@ module SwitchmanInstJobs
           remove_instance_variable(:@delayed_jobs_shards) if instance_variable_defined?(:@delayed_jobs_shards)
         end
 
-        def current(category = :primary)
-          if category == :delayed_jobs
-            active_shards[category] || super(:primary).delayed_jobs_shard
-          else
-            super
-          end
-        end
-
         def activate!(categories)
           if !@skip_delayed_job_auto_activation &&
-             !categories[:delayed_jobs] &&
-             categories[:primary] &&
-             categories[:primary] != active_shards[:primary]
+             !categories[::Delayed::Backend::ActiveRecord::AbstractJob] &&
+             categories[::ActiveRecord::Base] &&
+             categories[::ActiveRecord::Base] != ::Switchman::Shard.current(::ActiveRecord::Base)
             skip_delayed_job_auto_activation do
-              categories[:delayed_jobs] =
-                categories[:primary].delayed_jobs_shard
+              categories[::Delayed::Backend::ActiveRecord::AbstractJob] =
+                categories[::ActiveRecord::Base].delayed_jobs_shard
             end
           end
           super
@@ -104,11 +96,6 @@ module SwitchmanInstJobs
           @skip_delayed_job_auto_activation = was
         end
 
-        def create
-          db = ::Switchman::DatabaseServer.server_for_new_shard
-          db.create_new_shard
-        end
-
         def periodic_clear_shard_cache
           # TODO: make this configurable
           @timed_cache ||= TimedCache.new(-> { 60.to_i.seconds.ago }) do

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '3.2.6'.freeze
+  VERSION = '4.0.3'.freeze
 end

data/lib/switchman_inst_jobs.rb

@@ -5,6 +5,9 @@ module SwitchmanInstJobs
   cattr_accessor :delayed_jobs_shard_fallback
 
   def self.initialize_active_record
+    ::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(
+      ActiveRecord::ConnectionAdapters::ConnectionPool
+    )
     ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(
       ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
     )
@@ -32,6 +35,7 @@ module SwitchmanInstJobs
   end
 end
 
+require 'switchman_inst_jobs/active_record/connection_adapters/connection_pool'
 require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
 require 'switchman_inst_jobs/active_record/migration'
 require 'switchman_inst_jobs/delayed/settings'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.2.6
+  version: 4.0.3
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-09 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
@@ -50,34 +50,40 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '6.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.2'
+        version: '6.1'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '6.2'
 - !ruby/object:Gem::Dependency
   name: switchman
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -110,16 +116,16 @@ dependencies:
   name: diplomat
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.5.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.5.1
 - !ruby/object:Gem::Dependency
   name: newrelic_rpm
   requirement: !ruby/object:Gem::Requirement
@@ -182,70 +188,98 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '3.10'
 - !ruby/object:Gem::Dependency
   name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '5.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.6'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.1
+        version: '1.15'
 - !ruby/object:Gem::Dependency
   name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.1
+        version: '2.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.8.1
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: '0.21'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.18'
+        version: '0.21'
 - !ruby/object:Gem::Dependency
   name: wwtd
   requirement: !ruby/object:Gem::Requirement
@@ -289,6 +323,7 @@ files:
 - db/migrate/20151210162949_improve_max_concurrent.rb
 - db/migrate/20161206323555_add_back_default_string_limits_jobs.rb
 - db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb
+- db/migrate/20170308045401_add_delayed_jobs_shard_id_to_switchman_shards.rb
 - db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb
 - db/migrate/20190726154743_make_critical_columns_not_null.rb
 - db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb
@@ -303,8 +338,11 @@ files:
 - db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
 - db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
 - db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
+- db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
+- db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
+- lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
 - lib/switchman_inst_jobs/active_record/migration.rb
 - lib/switchman_inst_jobs/delayed/backend/base.rb