switchman-inst-jobs 3.0.5 → 3.2.10

data/lib/switchman_inst_jobs/delayed/worker/health_check.rb

@@ -18,22 +18,23 @@ module SwitchmanInstJobs
             ::Delayed::Settings.worker_health_check_config['service_name'] = original_service_name
           end
 
-          def reschedule_abandoned_jobs(call_super: false)
-            shards = ::Switchman::Shard.delayed_jobs_shards.to_a
-            call_super = shards.first if shards.length == 1
-            unless call_super == false
-              call_super.activate(:delayed_jobs) do
-                return munge_service_name(call_super) { super() }
+          def reschedule_abandoned_jobs
+            shard_ids = ::SwitchmanInstJobs::Delayed::Settings.configured_shard_ids
+            shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
+            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
+              dj_shard = ::Switchman::Shard.current(:delayed_jobs)
+              dj_shard.activate do
+                munge_service_name(dj_shard) do
+                  # because this rescheduling process is running on every host, we need
+                  # to make sure that it's functioning for each shard the current
+                  # host is programmed to interact with, but ONLY for those shards.
+                  # reading the config lets us iterate over any shards this host should
+                  # work with and lets us pick the correct service name to identify which
+                  # hosts are currently alive and valid via the health checks
+                  super()
+                end
               end
             end
-
-            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs], exception: :ignore) do
-              shard = ::Switchman::Shard.current(:delayed_jobs)
-              singleton = <<~SINGLETON
-                periodic: Delayed::Worker::HealthCheck.reschedule_abandoned_jobs:#{shard.id}
-              SINGLETON
-              delay(singleton: singleton).reschedule_abandoned_jobs(call_super: shard)
-            end
           end
         end
       end

data/lib/switchman_inst_jobs/engine.rb

@@ -19,13 +19,17 @@ module SwitchmanInstJobs
 
       # Ensure jobs get unblocked on the new shard if they exist
       ::Delayed::Worker.lifecycle.after(:perform) do |_worker, job|
-        if job.strand
+        if job.strand || job.singleton
+          column = job.strand ? :strand : :singleton
+
           ::Switchman::Shard.clear_cache
           ::Switchman::Shard.default.activate do
             current_job_shard = ::Switchman::Shard.lookup(job.shard_id).delayed_jobs_shard
             if current_job_shard != ::Switchman::Shard.current(:delayed_jobs)
               current_job_shard.activate(:delayed_jobs) do
-                j = ::Delayed::Job.where(strand: job.strand).next_in_strand_order.first
+                ::Delayed::Job.where(source: 'JobsMigrator::StrandBlocker', **{ column => job.try(column) }).delete_all
+
+                j = ::Delayed::Job.where(**{ column => job.try(column) }).next_in_strand_order.first
                 j.update_column(:next_in_strand, true) if j && !j.next_in_strand
               end
             end

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -1,7 +1,3 @@
-# Just disabling all the rubocop metrics for this file for now,
-# as it is a direct port-in of existing code
-
-# rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
 require 'set'
 require 'parallel'
 
@@ -39,16 +35,12 @@ module SwitchmanInstJobs
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
         target_shards.each do |target_shard, shards|
-          ::Switchman::Shard.where(id: shards).update_all(delayed_jobs_shard_id: target_shard, block_stranded: true)
+          updates = { delayed_jobs_shard_id: target_shard, block_stranded: true }
+          updates[:updated_at] = Time.zone.now if ::Switchman::Shard.column_names.include?('updated_at')
+          ::Switchman::Shard.where(id: shards).update_all(updates)
         end
         clear_shard_cache
 
-        # Wait a little over the 60 second in-process shard cache clearing
-        # threshold to ensure that all new stranded jobs are now being
-        # enqueued with next_in_strand: false
-        Rails.logger.debug('Waiting for caches to clear')
-        sleep(65) unless @skip_cache_wait
-
         ::Switchman::Shard.clear_cache
         # rubocop:disable Style/CombinableLoops
         # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
@@ -59,11 +51,33 @@ module SwitchmanInstJobs
         source_shards.each do |s|
           ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
         end
+        ensure_unblock_stranded_for(shard_map.map(&:first))
         # rubocop:enable Style/CombinableLoops
       end
 
-      def clear_shard_cache
+      # if :migrate_strands ran on any shards that fell into scenario 1, then
+      # block_stranded never got flipped, so do that now.
+      def ensure_unblock_stranded_for(shards)
+        shards = ::Switchman::Shard.where(id: shards, block_stranded: true).to_a
+        return unless shards.any?
+
+        ::Switchman::Shard.where(id: shards).update_all(block_stranded: false)
+        clear_shard_cache
+
+        # shards is an array of shard objects that is now stale cause block_stranded has been updated.
+        shards.map(&:delayed_jobs_shard).uniq.each do |dj_shard|
+          unblock_strands(dj_shard)
+        end
+      end
+
+      def clear_shard_cache(debug_message = nil)
         ::Switchman.cache.clear
+        Rails.logger.debug("Waiting for caches to clear #{debug_message}")
+        # Wait a little over the 60 second in-process shard cache clearing
+        # threshold to ensure that all new stranded jobs are now being
+        # enqueued with next_in_strand: false
+        # @skip_cache_wait is for spec usage only
+        sleep(65) unless @skip_cache_wait
       end
 
       # This method expects that all relevant shards already have block_stranded: true
@@ -75,7 +89,9 @@ module SwitchmanInstJobs
         migrate_everything
       end
 
-      def migrate_strands
+      def migrate_strands(batch_size: 1_000)
+        source_shard = ::Switchman::Shard.current(:delayed_jobs)
+
         # there are 4 scenarios to deal with here
         # 1) no running job, no jobs moved: do nothing
         # 2) running job, no jobs moved; create blocker with next_in_strand=false
@@ -84,97 +100,137 @@ module SwitchmanInstJobs
         #    those (= do nothing since it should already be false)
         # 4) no running job, jobs moved: set next_in_strand=true on the first of
         #    those (= do nothing since it should already be true)
+        handler = lambda { |scope, column, blocker_job_kwargs = {}, advisory_lock_cb = nil|
+          shard_map = build_shard_map(scope, source_shard)
+          shard_map.each do |(target_shard, source_shard_ids)|
+            shard_scope = scope.where(shard_id: source_shard_ids)
 
-        source_shard = ::Switchman::Shard.current(:delayed_jobs)
-        strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
-        shard_map = build_shard_map(strand_scope, source_shard)
-        shard_map.each do |(target_shard, source_shard_ids)|
-          shard_scope = strand_scope.where(shard_id: source_shard_ids)
-
-          # 1) is taken care of because it should not show up here in strands
-          strands = shard_scope.distinct.order(:strand).pluck(:strand)
-
-          target_shard.activate(:delayed_jobs) do
-            strands.each do |strand|
-              transaction_on([source_shard, target_shard]) do
-                this_strand_scope = shard_scope.where(strand: strand)
-                # we want to copy all the jobs except the one that is still running.
-                jobs_scope = this_strand_scope.where(locked_by: nil)
-
-                # 2) and part of 3) are taken care of here by creating a blocker
-                # job with next_in_strand = false. as soon as the current
-                # running job is finished it should set next_in_strand
-                # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
-                # Since we only unlock it on the new jobs queue *after* deleting from the original
-                # the lock ensures the blocker always gets unlocked
-                first = this_strand_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
-                if first
-                  first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
-                  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
-                  first_job.queue = first.queue
-                  first_job.tag = 'Kernel.sleep'
-                  first_job.source = 'JobsMigrator::StrandBlocker'
-                  first_job.max_attempts = 1
-                  # If we ever have jobs left over from 9999 jobs moves of a single shard,
-                  # something has gone terribly wrong
-                  first_job.strand_order_override = -9999
-                  first_job.save!
-                  # the rest of 3) is taken care of here
-                  # make sure that all the jobs moved over are NOT next in strand
-                  ::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil).
-                    update_all(next_in_strand: false)
-                end
+            # 1) is taken care of because it should not show up here in strands
+            values = shard_scope.distinct.order(column).pluck(column)
 
-                # 4) is taken care of here, by leaveing next_in_strand alone and
-                # it should execute on the new shard
-                batch_move_jobs(
-                  target_shard: target_shard,
-                  source_shard: source_shard,
-                  scope: jobs_scope
-                ) do |job, new_job|
-                  # This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
-                  new_job.strand_order_override = job.strand_order_override - 1
+            target_shard.activate(:delayed_jobs) do
+              values.each do |value|
+                transaction_on([source_shard, target_shard]) do
+                  source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+                    advisory_lock_cb&.call(value)
+                  end
+
+                  value_scope = shard_scope.where(**{ column => value })
+                  # we want to copy all the jobs except the one that is still running.
+                  jobs_scope = value_scope.where(locked_by: nil)
+
+                  # 2) and part of 3) are taken care of here by creating a blocker
+                  # job with next_in_strand = false. as soon as the current
+                  # running job is finished it should set next_in_strand
+                  # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
+                  # Since we only unlock it on the new jobs queue *after* deleting from the original
+                  # the lock ensures the blocker always gets unlocked
+                  first = value_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
+                  if first
+                    create_blocker_job(
+                      queue: first.queue,
+                      shard_id: first.shard_id,
+                      **{ column => value },
+                      **blocker_job_kwargs
+                    )
+
+                    # the rest of 3) is taken care of here
+                    # make sure that all the jobs moved over are NOT next in strand
+                    ::Delayed::Job.where(next_in_strand: true, locked_by: nil, **{ column => value }).
+                      update_all(next_in_strand: false)
+                  end
+
+                  # 4) is taken care of here, by leaving next_in_strand alone and
+                  # it should execute on the new shard
+                  batch_move_jobs(
+                    target_shard: target_shard,
+                    source_shard: source_shard,
+                    scope: jobs_scope,
+                    batch_size: batch_size
+                  ) do |job, new_job|
+                    # This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
+                    new_job.strand_order_override = job.strand_order_override - 1
+                  end
                 end
               end
             end
+          end
+        }
+
+        strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
+        singleton_scope = ::Delayed::Job.shard(source_shard).where('strand IS NULL AND singleton IS NOT NULL')
+        all_scope = ::Delayed::Job.shard(source_shard).where('strand IS NOT NULL OR singleton IS NOT NULL')
+
+        singleton_blocker_additional_kwargs = {
+          locked_at: DateTime.now,
+          locked_by: ::Delayed::Backend::Base::ON_HOLD_BLOCKER
+        }
 
+        quoted_function_name = ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
+        strand_advisory_lock_fn = lambda do |value|
+          ::Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock(#{quoted_function_name}('#{value}'))")
+        end
+
+        singleton_advisory_lock_fn = lambda do |value|
+          ::Delayed::Job.connection.execute(
+            "SELECT pg_advisory_xact_lock(#{quoted_function_name}('singleton:#{value}'))"
+          )
+        end
+
+        handler.call(strand_scope, :strand, {}, strand_advisory_lock_fn)
+        handler.call(singleton_scope, :singleton, singleton_blocker_additional_kwargs, singleton_advisory_lock_fn)
+
+        shard_map = build_shard_map(all_scope, source_shard)
+        shard_map.each do |(target_shard, source_shard_ids)|
+          target_shard.activate(:delayed_jobs) do
             updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
               update_all(block_stranded: false)
             # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
-            unless updated.zero?
-              clear_shard_cache
-              # Wait a little over the 60 second in-process shard cache clearing
-              # threshold to ensure that all new stranded jobs are now being
-              # enqueued with next_in_strand: false
-              Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
-              # for spec usage only
-              sleep(65) unless @skip_cache_wait
-            end
+            clear_shard_cache("(#{source_shard.id} -> #{target_shard.id})") unless updated.zero?
+
             ::Switchman::Shard.clear_cache
             # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
             # but actually could have run and we just didn't know it because we didn't know if they had jobs
             # on the source shard
-            target_shard.activate(:delayed_jobs) do
-              loop do
-                # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
-                # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
-                # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
-                # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
-                # batches
-                break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
-                  where.not(strand: nil).
-                  where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
-                    ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
-                    where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
-                    where('dj2.strand = delayed_jobs.strand').arel.exists.not
-                  ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
-              end
-            end
+            unblock_strands(target_shard)
           end
         end
       end
 
-      def migrate_everything
+      def unblock_strands(target_shard, batch_size: 10_000)
+        block_stranded_ids = ::Switchman::Shard.where(block_stranded: true).pluck(:id)
+        query = lambda { |column, scope|
+          ::Delayed::Job.
+            where(id: ::Delayed::Job.select("DISTINCT ON (#{column}) id").
+              where(scope).
+              where.not(shard_id: block_stranded_ids).
+              where(
+                ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
+                where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
+                where("dj2.#{column} = delayed_jobs.#{column}").arel.exists.not
+              ).
+              order(column, :strand_order_override, :id)).limit(batch_size)
+        }
+
+        target_shard.activate(:delayed_jobs) do
+          # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
+          # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
+          # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
+          # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
+          # batches
+
+          loop do
+            break if query.call(:strand, 'strand IS NOT NULL').update_all(next_in_strand: true).zero?
+          end
+
+          loop do
+            break if query.call(:singleton,
+                                'strand IS NULL AND singleton IS NOT NULL').update_all(next_in_strand: true).zero?
+          end
+        end
+      end
+
+      def migrate_everything(batch_size: 1_000)
         source_shard = ::Switchman::Shard.current(:delayed_jobs)
         scope = ::Delayed::Job.shard(source_shard).where('strand IS NULL')
 
@@ -183,13 +239,26 @@ module SwitchmanInstJobs
           batch_move_jobs(
             target_shard: target_shard,
             source_shard: source_shard,
-            scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil)
+            scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil),
+            batch_size: batch_size
           )
         end
       end
 
       private
 
+      def create_blocker_job(**kwargs)
+        first_job = ::Delayed::Job.create!(**kwargs, next_in_strand: false)
+        first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
+        first_job.tag = 'Kernel.sleep'
+        first_job.source = 'JobsMigrator::StrandBlocker'
+        first_job.max_attempts = 1
+        # If we ever have jobs left over from 9999 jobs moves of a single shard,
+        # something has gone terribly wrong
+        first_job.strand_order_override = -9999
+        first_job.save!
+      end
+
       def build_shard_map(scope, source_shard)
         shard_ids = scope.distinct.pluck(:shard_id)
 
@@ -204,10 +273,10 @@ module SwitchmanInstJobs
         shard_map
       end
 
-      def batch_move_jobs(target_shard:, source_shard:, scope:)
+      def batch_move_jobs(target_shard:, source_shard:, scope:, batch_size:)
         while scope.exists?
           # Adapted from get_and_lock_next_available in delayed/backend/active_record.rb
-          target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
+          target_jobs = scope.limit(batch_size).lock('FOR UPDATE SKIP LOCKED')
 
           query = source_shard.activate(:delayed_jobs) do
             "WITH limited_jobs AS (#{target_jobs.to_sql}) " \
@@ -266,7 +335,10 @@ module SwitchmanInstJobs
         connection = ::Delayed::Job.connection
         quoted_keys = keys.map { |k| connection.quote_column_name(k) }.join(', ')
 
-        connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
+        connection.execute 'DROP TABLE IF EXISTS delayed_jobs_bulk_copy'
+        connection.execute "CREATE TEMPORARY TABLE delayed_jobs_bulk_copy
+          (LIKE #{::Delayed::Job.quoted_table_name} INCLUDING DEFAULTS)"
+        connection.execute "COPY delayed_jobs_bulk_copy (#{quoted_keys}) FROM STDIN"
         records.map do |record|
           connection.raw_connection.put_copy_data("#{keys.map { |k| quote_text(record[k]) }.join("\t")}\n")
         end
@@ -278,6 +350,9 @@ module SwitchmanInstJobs
         rescue StandardError => e
           raise connection.send(:translate_exception, e, 'COPY FROM STDIN')
         end
+        connection.execute "INSERT INTO #{::Delayed::Job.quoted_table_name} (#{quoted_keys})
+          SELECT #{quoted_keys} FROM delayed_jobs_bulk_copy
+          ON CONFLICT (singleton) WHERE singleton IS NOT NULL AND locked_by IS NULL DO NOTHING"
         result.cmd_tuples
       end
 
@@ -295,5 +370,3 @@ module SwitchmanInstJobs
     end
   end
 end
-
-# rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -39,7 +39,14 @@ module SwitchmanInstJobs
 
       def unhold_jobs!
         self.jobs_held = false
-        save! if changed?
+        if changed?
+          save!
+          # Wait a little over the 60 second in-process shard cache clearing
+          # threshold to ensure that all new jobs are now being enqueued
+          # unlocked
+          Rails.logger.debug('Waiting for caches to clear')
+          sleep(65)
+        end
         delayed_jobs_shard.activate(:delayed_jobs) do
           ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id).
             in_batches(of: 10_000).

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '3.0.5'.freeze
+  VERSION = '3.2.10'.freeze
 end

data/lib/switchman_inst_jobs.rb

@@ -14,10 +14,6 @@ module SwitchmanInstJobs
     ::Delayed::Backend::ActiveRecord::Job.prepend(
       Delayed::Backend::Base
     )
-    ::Delayed::Backend::Redis::Job.prepend(
-      Delayed::Backend::Base
-    )
-    ::Delayed::Backend::Redis::Job.column :shard_id, :integer
     ::Delayed::Pool.prepend Delayed::Pool
     ::Delayed::Worker.prepend Delayed::Worker
     ::Delayed::Worker::HealthCheck.prepend Delayed::Worker::HealthCheck
@@ -38,6 +34,7 @@ end
 
 require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
 require 'switchman_inst_jobs/active_record/migration'
+require 'switchman_inst_jobs/delayed/settings'
 require 'switchman_inst_jobs/delayed/backend/base'
 require 'switchman_inst_jobs/delayed/message_sending'
 require 'switchman_inst_jobs/delayed/pool'

metadata

@@ -1,35 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.5
+  version: 3.2.10
 platform: ruby
 authors:
 - Bryan Petty
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-30 00:00:00.000000000 Z
+date: 2022-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 2.4.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: 2.4.9
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -107,7 +107,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: imperium
+  name: diplomat
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -260,7 +260,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
-description:
+description: 
 email:
 - bpetty@instructure.com
 executables: []
@@ -296,6 +296,19 @@ files:
 - db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb
 - db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb
 - db/migrate/20200825011002_add_strand_order_override.rb
+- db/migrate/20210809145804_add_n_strand_index.rb
+- db/migrate/20210812210128_add_singleton_column.rb
+- db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb
+- db/migrate/20210928174754_fix_singleton_condition_in_before_insert.rb
+- db/migrate/20210929204903_update_conflicting_singleton_function_to_use_index.rb
+- db/migrate/20211101190934_update_after_delete_trigger_for_singleton_index.rb
+- db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
+- db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
+- db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
+- db/migrate/20220127091200_fix_singleton_unique_constraint.rb
+- db/migrate/20220128084800_update_insert_trigger_for_singleton_unique_constraint_change.rb
+- db/migrate/20220128084900_update_delete_trigger_for_singleton_unique_constraint_change.rb
+- db/migrate/20220203063200_remove_old_singleton_index.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
@@ -303,6 +316,7 @@ files:
 - lib/switchman_inst_jobs/delayed/backend/base.rb
 - lib/switchman_inst_jobs/delayed/message_sending.rb
 - lib/switchman_inst_jobs/delayed/pool.rb
+- lib/switchman_inst_jobs/delayed/settings.rb
 - lib/switchman_inst_jobs/delayed/worker.rb
 - lib/switchman_inst_jobs/delayed/worker/health_check.rb
 - lib/switchman_inst_jobs/engine.rb
@@ -318,8 +332,10 @@ files:
 homepage: https://github.com/instructure/switchman-inst-jobs
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+metadata:
+  allowed_push_host: https://rubygems.org
+  rubygems_mfa_required: 'true'
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -327,7 +343,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.5'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -335,7 +351,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.4
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.
 test_files: []