switchman-inst-jobs 1.3.7 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bab0c63ffbb0a328830881dd3a7c4d87dcf4d257924540e3f817987cc0b02ca
-  data.tar.gz: 4efc1a1f73cd1c4e3343ebabb4aee21ce8b974b7191052616d6e1266e1547875
+  metadata.gz: ce47b09551ccb65d58da3f01b597ae32423282c1f74ba28f57cdc46a47aeb9cf
+  data.tar.gz: 137d3b0de54521971d62d94d33cfb4feaa1751cbb1520781397827f0ca8228be
 SHA512:
-  metadata.gz: 79109e3f82d473756cdb953c78b5e062c6f5bd4f262356c86c4502abe5aebff40a45679dea2ec530552d23b104fd70ab3f75f7686587d0da75c28ff97bc963fe
-  data.tar.gz: 39dda681ada80591021c3ca33df1061ae62307d94870fd6e4f5193cc361a4353a4b74f7ac86e0a4ab8bfd1fa4f4e800ac7556bbc6d4572b3ea314b5f4b551bf9
+  metadata.gz: f6df832cc871c8cb2ca0fc1c40740fbda65a98b1e815b8228b1cac2ce9599846da95c0068f68eab800116258ddc4baa028f7290374a8b852a8a84a1d11298ad5
+  data.tar.gz: ce795c604d25ad649e55acfeb3762bdd60f5418cdf48fbe3544bdd01310d7fffc6246b8146e011f4e58e94c06e130c096452add76a29c1a4210981378719849e

data/db/migrate/20200818130101_add_on_hold_to_switchman_shards.rb

@@ -0,0 +1,5 @@
+class AddOnHoldToSwitchmanShards < ActiveRecord::Migration[5.2]
+  def change
+    add_column :switchman_shards, :jobs_held, :bool, default: false
+  end
+end

data/db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb

@@ -0,0 +1,5 @@
+class AddBlockStrandedToSwitchmanShards < ActiveRecord::Migration[5.2]
+  def change
+    add_column :switchman_shards, :block_stranded, :bool, default: false
+  end
+end

data/lib/switchman_inst_jobs.rb

@@ -44,9 +44,11 @@ require 'switchman_inst_jobs/delayed/pool'
 require 'switchman_inst_jobs/delayed/worker'
 require 'switchman_inst_jobs/delayed/worker/health_check'
 require 'switchman_inst_jobs/engine'
+require 'switchman_inst_jobs/jobs_migrator'
 require 'switchman_inst_jobs/new_relic'
 require 'switchman_inst_jobs/shackles'
 require 'switchman_inst_jobs/switchman/database_server'
 require 'switchman_inst_jobs/switchman/default_shard'
 require 'switchman_inst_jobs/switchman/shard'
+require 'switchman_inst_jobs/timed_cache'
 require 'switchman_inst_jobs/yaml_extensions'

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -13,16 +13,32 @@ module SwitchmanInstJobs
       module Base
         module ClassMethods
           def enqueue(object, options = {})
+            ::Switchman::Shard.periodic_clear_shard_cache
+            current_shard = ::Switchman::Shard.current
             enqueue_options = options.merge(
-              current_shard: ::Switchman::Shard.current
+              current_shard: current_shard
             )
+            enqueue_job = -> { ::Shackles.activate(:master) { super(object, enqueue_options) } }
 
-            if ::ActiveRecord::Migration.open_migrations.positive?
-              ::Switchman::Shard.current.delayed_jobs_shard.activate(:delayed_jobs) do
-                super(object, enqueue_options)
-              end
+            # Another dj shard must be currently manually activated, so just use that
+            # In general this will only happen in unusual circumstances like tests
+            # also if migrations are running, always use the current shard's job shard
+            if ::ActiveRecord::Migration.open_migrations.zero? &&
+               current_shard.delayed_jobs_shard != ::Switchman::Shard.current(:delayed_jobs)
+              enqueue_job.call
             else
-              ::Shackles.activate(:master) { super(object, enqueue_options) }
+              ::Switchman::Shard.default.activate do
+                current_shard = ::Switchman::Shard.lookup(current_shard.id)
+                current_job_shard = current_shard.delayed_jobs_shard
+
+                if (options[:singleton] || options[:strand]) && current_shard.block_stranded
+                  enqueue_options[:next_in_strand] = false
+                end
+
+                current_job_shard.activate(:delayed_jobs) do
+                  enqueue_job.call
+                end
+              end
             end
           end
 
@@ -54,8 +70,15 @@ module SwitchmanInstJobs
         end
 
         def current_shard=(shard)
+          @current_shard = nil
           self.shard_id = shard.id
           self.shard_id = nil if shard.is_a?(::Switchman::DefaultShard)
+          # If jobs are held for a shard, enqueue new ones as held as well
+          return unless shard.jobs_held
+
+          self.locked_by = ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY
+          self.locked_at = ::Delayed::Job.db_time_now
+          self.attempts = ::Delayed::Backend::Base::ON_HOLD_COUNT
         end
 
         def invoke_job
@@ -64,12 +87,14 @@ module SwitchmanInstJobs
 
         def deserialize(source)
           raise ShardNotFoundError, shard_id unless current_shard
+
           current_shard.activate { super }
         rescue ::Switchman::ConnectionError, PG::ConnectionBad, PG::UndefinedTable
           # likely a missing shard with a stale cache
           current_shard.send(:clear_cache)
           ::Switchman::Shard.clear_cache
           raise ShardNotFoundError, shard_id unless ::Switchman::Shard.where(id: shard_id).exists?
+
           raise
         end
       end

data/lib/switchman_inst_jobs/delayed/message_sending.rb

@@ -3,6 +3,7 @@ module SwitchmanInstJobs
     module MessageSending
       def send_later_enqueue_args(method, _enqueue_args = {}, *args)
         return send(method, *args) if ::Switchman::DatabaseServer.creating_new_shard
+
         super
       end
     end

data/lib/switchman_inst_jobs/delayed/worker/health_check.rb

@@ -20,15 +20,20 @@ module SwitchmanInstJobs
 
           def reschedule_abandoned_jobs(call_super: false)
             shards = ::Switchman::Shard.delayed_jobs_shards
-            call_super = true if shards.length == 1
-            return munge_service_name(::Switchman::Shard.current(:delayed_jobs)) { super() } if call_super
+            call_super = shards.first if shards.length == 1
+            unless call_super == false
+              call_super.activate(:delayed_jobs) do
+                return munge_service_name(call_super) { super() }
+              end
+            end
 
             ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs], exception: :ignore) do
+              shard = ::Switchman::Shard.current(:delayed_jobs)
               singleton = <<~SINGLETON
-                periodic: Delayed::Worker::HealthCheck.reschedule_abandoned_jobs:#{::Switchman::Shard.current(:delayed_jobs).id}
+                periodic: Delayed::Worker::HealthCheck.reschedule_abandoned_jobs:#{shard.id}
               SINGLETON
               send_later_enqueue_args(
-                :reschedule_abandoned_jobs, { singleton: singleton }, call_super: true
+                :reschedule_abandoned_jobs, { singleton: singleton }, call_super: shard
               )
             end
           end

data/lib/switchman_inst_jobs/engine.rb

@@ -8,6 +8,30 @@ module SwitchmanInstJobs
 
     initializer 'sharding.delayed' do
       SwitchmanInstJobs.initialize_inst_jobs
+
+      ::Delayed::Worker.lifecycle.around(:work_queue_pop) do |worker, config, &block|
+        if config[:shard]
+          ::Switchman::Shard.lookup(config[:shard]).activate(:delayed_jobs) { block.call(worker, config) }
+        else
+          block.call(worker, config)
+        end
+      end
+
+      # Ensure jobs get unblocked on the new shard if they exist
+      ::Delayed::Worker.lifecycle.after(:perform) do |_worker, job|
+        if job.strand
+          ::Switchman::Shard.clear_cache
+          ::Switchman::Shard.default.activate do
+            current_job_shard = ::Switchman::Shard.lookup(job.shard_id).delayed_jobs_shard
+            if current_job_shard != ::Switchman::Shard.current(:delayed_jobs)
+              current_job_shard.activate(:delayed_jobs) do
+                j = ::Delayed::Job.where(strand: job.strand).next_in_strand_order.first
+                j.update_column(:next_in_strand, true) if j && !j.next_in_strand
+              end
+            end
+          end
+        end
+      end
     end
 
     initializer 'sharding.shackles', after: 'switchman.extend_shackles' do

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -0,0 +1,271 @@
+# Just disabling all the rubocop metrics for this file for now,
+# as it is a direct port-in of existing code
+
+# rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
+require 'set'
+require 'parallel'
+
+module SwitchmanInstJobs
+  class JobsMigrator
+    class << self
+      def add_before_move_callback(proc)
+        @before_move_callbacks ||= []
+        @before_move_callbacks << proc
+      end
+
+      def transaction_on(shards, &block)
+        return yield if shards.empty?
+
+        shard = shards.pop
+        current_shard = ::Switchman::Shard.current(:delayed_jobs)
+        shard.activate(:delayed_jobs) do
+          ::Delayed::Job.transaction do
+            current_shard.activate(:delayed_jobs) do
+              transaction_on(shards, &block)
+            end
+          end
+        end
+      end
+
+      def migrate_shards(shard_map)
+        source_shards = Set[]
+        shard_map.each do |(shard, target_shard)|
+          shard = ::Switchman::Shard.find(shard) unless shard.is_a?(::Switchman::Shard)
+          source_shards << shard.delayed_jobs_shard.id
+          # If target_shard is an int, it won't have an id, but we can just use it as is
+          shard.update(delayed_jobs_shard_id: target_shard.try(:id) || target_shard, block_stranded: true)
+        end
+
+        # Wait a little over the 60 second in-process shard cache clearing
+        # threshold to ensure that all new stranded jobs are now being
+        # enqueued with next_in_strand: false
+        Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
+        sleep(65) unless @skip_cache_wait
+
+        # TODO: 4 has been picked completely out of a hat.  We should make it configurable or something
+        Parallel.each(source_shards, in_processes: 4) do |s|
+          # Ensure the child processes don't share connections with the parent
+          Delayed::Pool.on_fork.call
+          ActiveRecord::Base.clear_all_connections!
+          s.activate(:delayed_jobs) { run }
+        end
+      end
+
+      # This method expects that all relevant shards already have block_stranded: true
+      # but otherwise jobs can be running normally
+      def run
+        # Ensure this is never run with a dirty in-memory shard cache
+        ::Switchman::Shard.clear_cache
+        migrate_strands
+        migrate_everything
+      end
+
+      def migrate_strands
+        # there are 4 scenarios to deal with here
+        # 1) no running job, no jobs moved: do nothing
+        # 2) running job, no jobs moved; create blocker with next_in_strand=false
+        #    to prevent new jobs from immediately executing
+        # 3) running job, jobs moved; set next_in_strand=false on the first of
+        #    those (= do nothing since it should already be false)
+        # 4) no running job, jobs moved: set next_in_strand=true on the first of
+        #    those (= do nothing since it should already be true)
+
+        source_shard = ::Switchman::Shard.current(:delayed_jobs)
+        strand_scope = ::Delayed::Job.shard(source_shard).where('strand IS NOT NULL')
+        shard_map = build_shard_map(strand_scope, source_shard)
+        shard_map.each do |(target_shard, source_shard_ids)|
+          shard_scope = strand_scope.where(shard_id: source_shard_ids)
+
+          # 1) is taken care of because it should not show up here in strands
+          strands = shard_scope.distinct.order(:strand).pluck(:strand)
+
+          target_shard.activate(:delayed_jobs) do
+            strands.each do |strand|
+              transaction_on([source_shard, target_shard]) do
+                this_strand_scope = shard_scope.where(strand: strand)
+                # we want to copy all the jobs except the one that is still running.
+                jobs_scope = this_strand_scope.where(locked_by: nil)
+
+                # 2) and part of 3) are taken care of here by creating a blocker
+                # job with next_in_strand = false. as soon as the current
+                # running job is finished it should set next_in_strand
+                # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
+                # Since we only unlock it on the new jobs queue *after* deleting from the original
+                # the lock ensures the blocker always gets unlocked
+                first = this_strand_scope.where('locked_by IS NOT NULL').next_in_strand_order.lock.first
+                if first
+                  first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
+                  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, [0])
+                  first_job.queue = first.queue
+                  first_job.tag = 'Kernel.sleep'
+                  first_job.source = 'JobsMigrator::StrandBlocker'
+                  first_job.max_attempts = 1
+                  # If we ever have jobs left over from 9999 jobs moves of a single shard,
+                  # something has gone terribly wrong
+                  first_job.strand_order_override = -9999
+                  first_job.save!
+                  # the rest of 3) is taken care of here
+                  # make sure that all the jobs moved over are NOT next in strand
+                  ::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil)
+                    .update_all(next_in_strand: false)
+                end
+
+                # 4) is taken care of here, by leaveing next_in_strand alone and
+                # it should execute on the new shard
+                batch_move_jobs(
+                  target_shard: target_shard,
+                  source_shard: source_shard,
+                  scope: jobs_scope
+                ) do |job, new_job|
+                  # This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
+                  new_job.strand_order_override = job.strand_order_override - 1
+                end
+              end
+            end
+
+            ::Switchman::Shard.find(source_shard_ids).each do |shard|
+              shard.update(block_stranded: false)
+            end
+            # Wait a little over the 60 second in-process shard cache clearing
+            # threshold to ensure that all new stranded jobs are now being
+            # enqueued with next_in_strand: false
+            Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
+            # for spec usage only
+            sleep(65) unless @skip_cache_wait
+            # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
+            # but actually could have run and we just didn't know it because we didn't know if they had jobs
+            # on the source shard
+            # rubocop:disable Layout/LineLength
+            strands_to_unblock = shard_scope.where.not(source: 'JobsMigrator::StrandBlocker')
+              .distinct
+              .where("NOT EXISTS (SELECT 1 FROM #{::Delayed::Job.quoted_table_name} dj2 WHERE delayed_jobs.strand=dj2.strand AND next_in_strand)")
+              .pluck(:strand)
+            # rubocop:enable Layout/LineLength
+            strands_to_unblock.each do |strand|
+              Delayed::Job.where(strand: strand).next_in_strand_order.first.update_attribute(:next_in_strand, true)
+            end
+          end
+        end
+      end
+
+      def migrate_everything
+        source_shard = ::Switchman::Shard.current(:delayed_jobs)
+        scope = ::Delayed::Job.shard(source_shard).where('strand IS NULL')
+
+        shard_map = build_shard_map(scope, source_shard)
+        shard_map.each do |(target_shard, source_shard_ids)|
+          batch_move_jobs(
+            target_shard: target_shard,
+            source_shard: source_shard,
+            scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil)
+          )
+        end
+      end
+
+      private
+
+      def build_shard_map(scope, source_shard)
+        shard_ids = scope.distinct.pluck(:shard_id)
+
+        shard_map = {}
+        ::Switchman::Shard.find(shard_ids).each do |shard|
+          next if shard.delayed_jobs_shard == source_shard
+
+          shard_map[shard.delayed_jobs_shard] ||= []
+          shard_map[shard.delayed_jobs_shard] << shard.id
+        end
+
+        shard_map
+      end
+
+      def batch_move_jobs(target_shard:, source_shard:, scope:)
+        while scope.exists?
+          # Adapted from get_and_lock_next_available in delayed/backend/active_record.rb
+          target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
+
+          query = "WITH limited_jobs AS (#{target_jobs.to_sql}) " \
+                  "UPDATE #{::Delayed::Job.quoted_table_name} " \
+                  "SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)}, " \
+                  "locked_at = #{::Delayed::Job.connection.quote(::Delayed::Job.db_time_now)} "\
+                  "FROM limited_jobs WHERE limited_jobs.id=#{::Delayed::Job.quoted_table_name}.id " \
+                  "RETURNING #{::Delayed::Job.quoted_table_name}.*"
+
+          jobs = source_shard.activate(:delayed_jobs) { ::Delayed::Job.find_by_sql(query) }
+          new_jobs = jobs.map do |job|
+            new_job = job.dup
+            new_job.shard = target_shard
+            new_job.created_at = job.created_at
+            new_job.updated_at = job.updated_at
+            new_job.locked_at = nil
+            new_job.locked_by = nil
+            yield(job, new_job) if block_given?
+            @before_move_callbacks&.each do |proc|
+              proc.call(
+                old_job: job,
+                new_job: new_job
+              )
+            end
+            new_job
+          end
+          transaction_on([source_shard, target_shard]) do
+            target_shard.activate(:delayed_jobs) do
+              bulk_insert_jobs(new_jobs)
+            end
+            source_shard.activate(:delayed_jobs) do
+              ::Delayed::Job.delete(jobs)
+            end
+          end
+        end
+      end
+
+      # This is adapted from the postgreql adapter in canvas-lms
+      # Once we stop supporting rails 5.2 we can just use insert_all from activerecord
+      def bulk_insert_jobs(objects)
+        records = objects.map do |object|
+          object.attributes.map do |(name, value)|
+            next if name == ::Delayed::Job.primary_key
+
+            if (type = ::Delayed::Job.attribute_types[name]).is_a?(::ActiveRecord::Type::Serialized)
+              value = type.serialize(value)
+            end
+            [name, value]
+          end.compact.to_h
+        end
+        return if records.length.zero?
+
+        keys = records.first.keys
+
+        connection = ::Delayed::Job.connection
+        quoted_keys = keys.map { |k| connection.quote_column_name(k) }.join(', ')
+
+        connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
+        records.map do |record|
+          connection.raw_connection.put_copy_data(keys.map { |k| quote_text(record[k]) }.join("\t") + "\n")
+        end
+        connection.clear_query_cache
+        connection.raw_connection.put_copy_end
+        result = connection.raw_connection.get_result
+        begin
+          result.check
+        rescue StandardError => e
+          raise connection.send(:translate_exception, e, 'COPY FROM STDIN')
+        end
+        result.cmd_tuples
+      end
+
+      # See above comment...
+      def quote_text(value)
+        if value.nil?
+          '\\N'
+        elsif value.is_a?(::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array::Data)
+          quote_text(encode_array(value))
+        else
+          hash = { "\n" => '\\n', "\r" => '\\r', "\t" => '\\t', '\\' => '\\\\' }
+          value.to_s.gsub(/[\n\r\t\\]/) { |c| hash[c] }
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength

data/lib/switchman_inst_jobs/switchman/database_server.rb

@@ -3,6 +3,7 @@ module SwitchmanInstJobs
     module DatabaseServer
       def delayed_jobs_shard(shard = nil)
         return shard if config[:delayed_jobs_shard] == 'self'
+
         dj_shard =
           config[:delayed_jobs_shard] &&
           ::Switchman::Shard.lookup(config[:delayed_jobs_shard])

data/lib/switchman_inst_jobs/switchman/default_shard.rb

@@ -4,6 +4,14 @@ module SwitchmanInstJobs
       def delayed_jobs_shard
         self
       end
+
+      def jobs_held
+        false
+      end
+
+      def block_stranded
+        false
+      end
     end
   end
 end

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -17,6 +17,51 @@ module SwitchmanInstJobs
         database_server&.delayed_jobs_shard(self)
       end
 
+      # Adapted from hold/unhold methods in base delayed jobs base
+      # Wait is required to be able to safely move jobs
+      def hold_jobs!(wait: false)
+        self.jobs_held = true
+        save! if changed?
+        delayed_jobs_shard.activate(:delayed_jobs) do
+          lock_jobs_for_hold
+        end
+        return unless wait
+
+        delayed_jobs_shard.activate(:delayed_jobs) do
+          while ::Delayed::Job.where(shard_id: id)
+              .where.not(locked_at: nil)
+              .where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
+            sleep 10
+            lock_jobs_for_hold
+          end
+        end
+      end
+
+      def unhold_jobs!
+        self.jobs_held = false
+        save! if changed?
+        delayed_jobs_shard.activate(:delayed_jobs) do
+          ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id)
+            .in_batches(of: 10_000)
+            .update_all(
+              locked_by: nil,
+              locked_at: nil,
+              attempts: 0,
+              failed_at: nil
+            )
+        end
+      end
+
+      private
+
+      def lock_jobs_for_hold
+        ::Delayed::Job.where(locked_at: nil, shard_id: id).in_batches(of: 10_000).update_all(
+          locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY,
+          locked_at: ::Delayed::Job.db_time_now,
+          attempts: ::Delayed::Backend::Base::ON_HOLD_COUNT
+        )
+      end
+
       module ClassMethods
         def clear_cache
           super
@@ -57,6 +102,14 @@ module SwitchmanInstJobs
           db.create_new_shard
         end
 
+        def periodic_clear_shard_cache
+          # TODO: make this configurable
+          @timed_cache ||= TimedCache.new(-> { 60.to_i.seconds.ago }) do
+            ::Switchman::Shard.clear_cache
+          end
+          @timed_cache.clear
+        end
+
         def delayed_jobs_shards
           unless instance_variable_defined?(:@delayed_jobs_shards)
             # re-entrancy protection
@@ -76,6 +129,7 @@ module SwitchmanInstJobs
 
               db_dj_shards = ::Switchman::DatabaseServer.all.map do |db|
                 next db.shards.to_a if db.config[:delayed_jobs_shard] == 'self'
+
                 db.delayed_jobs_shard
               end.compact.flatten.uniq # yes, all three
 

data/lib/switchman_inst_jobs/timed_cache.rb

@@ -0,0 +1,19 @@
+module SwitchmanInstJobs
+  class TimedCache
+    def initialize(timeout, &block)
+      @timeout = timeout
+      @block = block
+      @cached_at = Time.zone.now
+    end
+
+    def clear(force = false)
+      if force || @cached_at < @timeout.call
+        @block.call
+        @cached_at = Time.zone.now
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '1.3.7'.freeze
+  VERSION = '1.5.1'.freeze
 end

data/lib/switchman_inst_jobs/yaml_extensions.rb

@@ -5,7 +5,7 @@ module SwitchmanInstJobs
         if object.tag == '!ruby/ActiveRecord:Switchman::Shard'
           ::Switchman::Shard.lookup(object.value) ||
             raise(Delayed::Backend::RecordNotFound,
-              "Couldn't find Switchman::Shard with id #{object.value.inspect}")
+                  "Couldn't find Switchman::Shard with id #{object.value.inspect}")
         else
           super
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 1.3.7
+  version: 1.5.1
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-21 00:00:00.000000000 Z
+date: 2020-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
@@ -16,27 +16,41 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.1
+        version: '0.16'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.1
+        version: '0.16'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
@@ -46,7 +60,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4.2'
+        version: '5.2'
     - - "<"
       - !ruby/object:Gem::Version
         version: '6.1'
@@ -56,20 +70,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.7
+        version: '1.14'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.9.7
+        version: '1.14'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '1.15'
+        version: '1.16'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -132,14 +146,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -160,14 +174,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '12'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12.0'
+        version: '12'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -202,42 +216,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.57.2
+        version: 0.79.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.79.0
+- !ruby/object:Gem::Dependency
+  name: rubocop-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.4.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.57.2
+        version: 2.4.2
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '0.18'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.14'
+        version: '0.18'
 - !ruby/object:Gem::Dependency
   name: wwtd
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '1.4'
 description: 
 email:
 - bpetty@instructure.com
@@ -247,6 +275,8 @@ extra_rdoc_files: []
 files:
 - db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb
 - db/migrate/20180628153808_set_search_paths_on_functions.rb
+- db/migrate/20200818130101_add_on_hold_to_switchman_shards.rb
+- db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
@@ -257,11 +287,13 @@ files:
 - lib/switchman_inst_jobs/delayed/worker.rb
 - lib/switchman_inst_jobs/delayed/worker/health_check.rb
 - lib/switchman_inst_jobs/engine.rb
+- lib/switchman_inst_jobs/jobs_migrator.rb
 - lib/switchman_inst_jobs/new_relic.rb
 - lib/switchman_inst_jobs/shackles.rb
 - lib/switchman_inst_jobs/switchman/database_server.rb
 - lib/switchman_inst_jobs/switchman/default_shard.rb
 - lib/switchman_inst_jobs/switchman/shard.rb
+- lib/switchman_inst_jobs/timed_cache.rb
 - lib/switchman_inst_jobs/version.rb
 - lib/switchman_inst_jobs/yaml_extensions.rb
 homepage: https://github.com/instructure/switchman-inst-jobs
@@ -276,7 +308,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.3'
+      version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="